diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 205 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 62 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 1 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 228 |
4 files changed, 436 insertions, 60 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 967dc25..42a0f2a 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -140,6 +140,7 @@ static inline bool gsp_phys_gr_copywit_enabled(void) return READ_ONCE(gsp_phys_gr_copyout_dump_param); } + /* * r405 late-write probes (the r405 "partial write" class: the scale launch * of streams2 wrote 1 of its 32 expected in-place pages and every other @@ -16472,7 +16473,7 @@ static bool gsp_phys_rmcall_gr_init_emitted; * stale-QMD execution) landed after its launch was shipped. */ static u32 gsp_phys_gr_prev_out_bi; -static u32 gsp_phys_gr_prev_out_csum; +static u64 gsp_phys_gr_prev_out_csum; static bool gsp_phys_gr_prev_out_valid; /* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged @@ -17389,6 +17390,54 @@ static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) } /* + * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern + * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan). + * Verifies by content where a lost engine store actually landed: the chunk + * is the only bulk memory the operand window PTEs map, so a store that + * executed but is absent from the writer's tracked pages must be findable + * here (a stale-PTE landing) or nowhere (a dropped store). + */ +static int gsp_phys_gr_scan_set(const char *val, + const struct kernel_param *kp) +{ + struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk(); + u64 pattern = 0; + u32 i, hits = 0; + + if (kstrtoull(val, 0, &pattern)) + return -EINVAL; + if (!c->pages) + return -ENODEV; + for (i = 0; i < c->pages; i++) { + const u8 *p = gsp_phys_chunk_page_va(c, i); + u32 off; + + if (!p) + continue; + for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) { + if (get_unaligned_le64(p + off) == pattern) { + if (hits < 32) + pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n", + pattern, i, off, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)i << 12) + off); + hits++; + } + } + } + pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n", + pattern, hits, c->pages); + return 0; +} + +static const struct kernel_param_ops gsp_phys_gr_scan_ops = { + .set = gsp_phys_gr_scan_set, +}; +module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200); +MODULE_PARM_DESC(gsp_phys_gr_scan, + "Scan the GR data-bounce chunk for an 8-byte pattern (hex)"); + +/* * B-2: page index of a relocated operand VA inside the engine-visible data * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations * and their MAIL_SLOT bounds. Grown region: VAs from @@ -17838,7 +17887,7 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, */ static noinline u32 gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked, u64 *hash_out) + const u64 *csum_in, u32 *n_tracked, u64 *hash_out) { static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); @@ -17884,15 +17933,24 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, *n_tracked = n_all; for (j = 0; j < n_all; j++) { const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); - u32 ci, csum = 0; + u64 fp; if (!src) continue; - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != csum_in[checked[j]]) + /* + * r452: FNV-1a per page, not a 32-bit additive sum. The + * add-sum was PROVABLY blind to a power-of-two in-place + * scale: doubling every word of a 4 KB page shifts each + * exponent by +1, i.e. +2^23 per word, and 1024 words give + * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum + * bit-identical, the page classified "unchanged", and the + * client read the pre-scale values (the deterministic + * streams2/scale class). FNV-1a has no such structure. + */ + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp != csum_in[checked[j]]) changed++; - hash ^= csum; + hash ^= fp; hash *= 1099511628211ULL; } if (hash_out) @@ -17928,7 +17986,7 @@ static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag) static noinline u32 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked) + const u64 *csum_in, u32 *n_tracked) { u32 n_all = 0, iter; u32 changed = 0; @@ -18082,7 +18140,7 @@ static u32 gsp_phys_gr_idem_n; static u64 gsp_phys_gr_launch_fp(const u32 *qmd, const struct gsp_phys_gr_submit *req, - const u32 *csum_in) + const u64 *csum_in) { u64 h = 1469598103934665603ULL; u32 i; @@ -18160,6 +18218,22 @@ static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, } /* + * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero + * filter. It is never a change-detection fingerprint any more: doubling every + * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0 + * mod 2^32), which silently classified an in-place x2 scale as "no engine + * store" and shipped stale values (the deterministic streams2 class). + */ +static inline u32 gsp_phys_gr_page_sum32(const u8 *p) +{ + u32 ci, csum = 0; + + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(p + ci); + return csum; +} + +/* * COPYWIT r226 zero filter: an all-zero page is never a meaningful alias * partner. A fresh client's fill stages long runs of zero pages * (uninitialized cudaMalloc buffers), and consecutive zero pages produced @@ -18205,7 +18279,7 @@ static u32 gsp_phys_gr_in_census_log; * again). Returns the number of pages staged. */ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, - u32 *csum_in, u64 *staged_hpa) + u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); struct { @@ -18242,7 +18316,9 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(dst + ci); - csum_in[bi] = csum; + /* r452: the change baseline is the FNV-1a fingerprint (the + * add-sum above survives only as the all-zero test). */ + csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); zero = gsp_phys_gr_page_all_zero(csum, dst); if (!zero && gsp_phys_gr_copywit_enabled()) { u32 j; @@ -18291,7 +18367,7 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, * retry, so a retried launch starts from the guest inputs in both sets. * Returns the number of pages staged. */ -static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) +static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 gi, copied = 0; @@ -18299,7 +18375,6 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi); - u32 ci, csum = 0; if (!src || !dst) continue; @@ -18309,9 +18384,9 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, "gr-bounce-copy-in"); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - csum_in[gsp_phys_gr_gap[gi].bi] = csum; + /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */ + csum_in[gsp_phys_gr_gap[gi].bi] = + gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); copied++; } dma_wmb(); @@ -18574,6 +18649,7 @@ static u32 gsp_phys_m91_fault_start; static u32 gsp_phys_m91_cap_used; static bool gsp_phys_m91_cap_suppressed; static bool gsp_phys_m91_icache_log; +static bool gsp_phys_r449_ccache_log; static bool gsp_phys_gr_rel_membar_log; static noinline void gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, @@ -18906,7 +18982,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * stack). Safe as static: the only caller is the single-inflight * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; /* * r378: which guest page's content each bounce slot holds this * launch (set by the copy-in fills, cleared per submission). The @@ -19084,6 +19160,32 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } } /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default + * 0 = off): the same cache-control word's bit 31 (the V01_06-layout + * SHADER_CONSTANT_CACHE position) is never set by the captured guest + * QMDs. The forwarded param page is rewritten in place between launches + * (the two-bank win0 pool), so a stale constant-cache line can serve + * the PREVIOUS launch's parameters and the current launch's output + * pages stay untouched - the post-dispatch no-store signature. Mode 1 + * sets bit 31; mode 2 forces the whole invalidate set (bits 24..31). + * Only when the QMD carries the driver's standard cache-control + * signature (bits 26..29) so an unknown QMD revision is never touched. + */ + if (gsp_phys_gr_ccache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u) { + u32 mode = gsp_phys_gr_ccache_inv_value(); + + if (mode == 2) + qmd[5] |= 0xff000000u; + else + qmd[5] |= 1u << 31; + if (!gsp_phys_r449_ccache_log) { + gsp_phys_r449_ccache_log = true; + pr_info("vgsp-gr: r449 constant-cache invalidate mode=%u applied (w5=0x%08x)\n", + mode, qmd[5]); + } + } + /* * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar): force the * forwarded QMD's RELEASE_MEMBAR_TYPE to FE_SYSMEMBAR (QMDV00_06 * word 11 bit 14) so the front-end flushes its pending system-memory @@ -21414,14 +21516,12 @@ gr_page_unmapped: const u8 *pp = gsp_phys_chunk_page_va( gsp_phys_gr_data_chunk(), gsp_phys_gr_prev_out_bi); - u32 pc = 0, pi; + u64 pc; if (pp) { - for (pi = 0; pi < GSP_PHYS_PAGE_SIZE; - pi += 4) - pc += get_unaligned_le32(pp + pi); + pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE); if (pc != gsp_phys_gr_prev_out_csum) - pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u csum %08x -> %08x (late store after its launch shipped)\n", + pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n", gsp_phys_gr_prev_out_bi, gsp_phys_gr_prev_out_csum, pc); @@ -21518,7 +21618,7 @@ gr_page_unmapped: for (wbi = 0; wbi < req->n_data; wbi++) { u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; - u32 ci, csum = 0; + u64 fp; if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) @@ -21539,9 +21639,8 @@ gr_page_unmapped: aliased++; continue; } - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum == bounce_csum_in[bi]) { + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { /* * r378: the engine did not touch this page, * so the guest page still holds exactly this @@ -21550,6 +21649,37 @@ gr_page_unmapped: * stale-content corruption at worst - skip. */ unchanged++; + /* + * r452: witness the UNCHANGED page's chunk + * content (first 4 words) and the guest + * page's, bounded per launch. This is the + * only place the two can be compared for a + * page the csum rule classified as "engine + * did not touch" - it separates "the store + * never landed in the chunk" from "the + * store landed but the csum rule missed + * it" (or "the reader resolves a different + * page than the writer wrote"). + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + const u8 *gp = req->data[wbi].hpa ? + (const u8 *)__va(req->data[wbi].hpa) : + NULL; + + outwit++; + pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n", + bi, req->data[wbi].va, fp, + bounce_csum_in[bi], + get_unaligned_le32(src), + get_unaligned_le32(src + 4), + get_unaligned_le32(src + 8), + get_unaligned_le32(src + 12), + gp ? get_unaligned_le32(gp) : 0, + gp ? get_unaligned_le32(gp + 4) : 0, + gp ? get_unaligned_le32(gp + 8) : 0, + gp ? get_unaligned_le32(gp + 12) : 0); + } continue; } { @@ -21600,13 +21730,13 @@ gr_page_unmapped: &rap, &rval); outwit++; - pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%08x post=%08x res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", req->data[wbi].va, req->data[wbi].hpa, bi, f8, (u64)get_unaligned_le64( src + GSP_PHYS_PAGE_SIZE - 8), - bounce_csum_in[bi], csum, + bounce_csum_in[bi], fp, rhave ? "ledger" : "none", rpa, rap, rval, (void *)gsp_sema_gr_shadow_base()); @@ -21621,7 +21751,8 @@ gr_page_unmapped: * evidence). */ if (gsp_phys_gr_copywit_enabled() && - !gsp_phys_gr_page_all_zero(csum, src)) { + !gsp_phys_gr_page_all_zero( + gsp_phys_gr_page_sum32(src), src)) { u32 wi; for (wi = 0; wi < gsp_phys_gr_outwit_n; @@ -21657,7 +21788,7 @@ gr_page_unmapped: */ if (changed == 0) { gsp_phys_gr_prev_out_bi = bi; - gsp_phys_gr_prev_out_csum = csum; + gsp_phys_gr_prev_out_csum = fp; gsp_phys_gr_prev_out_valid = true; } changed++; @@ -21738,8 +21869,7 @@ gr_page_unmapped: if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; wp = gsp_phys_chunk_page_va(dchunk, bi); - for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) - wc += get_unaligned_le32(wp + k); + (void)k; (void)wc; pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", bi, req->data[wi].reloc_va, @@ -21747,7 +21877,9 @@ gr_page_unmapped: get_unaligned_le32(wp + 4), get_unaligned_le32(wp + 8), get_unaligned_le32(wp + 12), - wc != bounce_csum_in[bi]); + gsp_phys_gr_fnv1a64( + wp, GSP_PHYS_PAGE_SIZE) != + bounce_csum_in[bi]); } } pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", @@ -21770,7 +21902,7 @@ gr_page_unmapped: for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { u32 bi = gsp_phys_gr_gap[gi].bi; const u8 *src = gsp_phys_chunk_page_va(dchunk, bi); - u32 ci, csum = 0; + u64 fp; if (!src) continue; @@ -21783,9 +21915,8 @@ gr_page_unmapped: gskipped++; continue; } - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum == bounce_csum_in[bi]) { + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { /* Engine did not touch it: nothing to ship. */ gunchanged++; continue; diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index b61929e..74062dd 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -734,20 +734,30 @@ int gsp_phys_rmcall_gr_qmdwin_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value); /* - * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod; default 0). - * When set, a forwarded GR launch ignores the resolved guest release page and - * points QMD release0 at the module window-0 semaphore VA - * (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host VA. - * Discriminant for "engine completed but the shadow release write was lost" - * vs "engine never completed": with the module release, a completed QMD is - * always observable. Also makes the existing module-release fallback - * observable (it previously set no poll source, so it could never complete). - */ -static int gsp_phys_rmcall_gr_rel_mod_param; + * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod). + * + * DEFAULT 1 since r453. When set, a forwarded GR launch ignores the resolved + * guest release page and points QMD release0 at the module window-0 semaphore + * VA (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host + * VA; the GUEST-VISIBLE release word is authored by the host only on the + * completion edge - after the retire wait and after the bounce copy-out has + * shipped the engine's stores to the guest pages. Evidence (r453, the FNV + * change-detection fix on top): + * - with 0: the guest's release could be satisfied by the engine's own + * early write while the host copy-out was still pending, so a client's + * stream-ordered readback (streams2's c2, memcpy_roundtrip's memset + * phase, streams_events' chain) read PRE-launch content: 11/15 battery. + * - with 1: 13/15 battery (streams_events and memcpy_roundtrip flipped + * green), no stalls across the full battery, mail+saxpy clean. + * The r414 stall hole (a launch that never reached the hold edge left the + * guest word unauthored) is closed by the r420b per-outcome authoring path; + * one-line rollback: echo 0 > .../parameters/gsp_phys_rmcall_gr_rel_mod. + */ +static int gsp_phys_rmcall_gr_rel_mod_param = 1; module_param_named(gsp_phys_rmcall_gr_rel_mod, gsp_phys_rmcall_gr_rel_mod_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_rel_mod, - "Force the forwarded GR QMD release0 to the module semaphore (default 0; 1 = m64 engine-completion discriminant)"); + "Force the forwarded GR QMD release0 to the module semaphore; guest release authored only on the post-copy-out completion edge (default 1 since r453; 0 = historical engine-authored guest release)"); int gsp_phys_rmcall_gr_rel_mod_value(void) { @@ -1198,6 +1208,36 @@ int gsp_phys_gr_wedge_rearm_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_rearm_value); /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default 0). + * The forwarded QMD's cache-control block (QMDV02_03 word 5) carries the + * per-cache invalidate bits; the captured guest QMDs always set bits 26..29 + * (texture header/sampler/data + shader DATA) and the M91 fix adds bit 30 + * (INSTRUCTION), but bit 31 (SHADER CONSTANT - where a kernel's launch + * parameters live) is never set. The forwarded launch's param page is + * rewritten in place between launches (the two-bank win0 pool), so a stale + * constant-cache line can serve the PREVIOUS launch's parameters - the + * kernel then computes with the previous launch's pointers/sizes and the + * current launch's tracked output pages stay untouched ("no stores"). + * Mode: 0 = off, 1 = set bit 31 only, 2 = force the full 0xff000000 + * invalidate set. Read at submission time, so it can be A/B'd at runtime. + */ +static int gsp_phys_gr_ccache_inv_param; +module_param_named(gsp_phys_gr_ccache_inv, gsp_phys_gr_ccache_inv_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_ccache_inv, + "r449: forwarded-QMD shader-constant-cache invalidate: 0=off, 1=bit31, 2=all invalidate bits (default 0)"); + +unsigned int gsp_phys_gr_ccache_inv_value(void) +{ + if (gsp_phys_gr_ccache_inv_param < 0) + return 0; + if (gsp_phys_gr_ccache_inv_param > 2) + return 2; + return (unsigned int)gsp_phys_gr_ccache_inv_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_ccache_inv_value); + +/* * r410 wedge quiet window (gsp_phys_gr_wedge_quiet_ms; default 40000, clamped * [0,120000]; 0 disables). The sticky per-lane no-execute wedge (100% of * launches no-write while kept busy) is proven to CLEAR when the lane stays diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 46d841d..36c2220 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -403,6 +403,7 @@ int gsp_phys_rmcall_gr_cwd_slot_value(void); * attempt counter (default 3, clamped [0,4]). */ int gsp_phys_gr_nowrite_retries_value(void); int gsp_phys_gr_wedge_rearm_value(void); +unsigned int gsp_phys_gr_ccache_inv_value(void); /* STALLFREE: absolute per-launch GR submit wall-clock budget in ms, clamped * [10000,600000]; 0 = only the built-in per-ladder bounds (default 60000). diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b90c948..28e5906 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -486,6 +486,12 @@ struct svc_vaspace { u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ + unsigned long stamp; /* r459: last registration/use; LRU evict + * when the table is full (a stale entry from + * a dead client otherwise blocks every new + * client after the 8th registration: the + * mid-boot walk failure that made the 10th+ + * client's copies fail rc=-2) */ }; struct svc_lat { @@ -819,6 +825,7 @@ struct sema_svc { u64 pushtrace_all; /* fail4: de-windowed push decode * trace (every push, bounded) */ u64 opadd_log; /* fail4: every op appended */ + u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */ u64 dmatrace_log; /* fail4: denied-transfer trace */ u64 pend_dbg_log; /* fail4 r305: pending/hold witness */ u64 snap_gate_log; /* fail4 r311: capture-gate witness */ @@ -1599,12 +1606,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } + /* + * r459: the table is 8 slots and the guest driver does not always + * send a teardown for a client's VA space (observed: 23 distinct + * registrations, 0 drops in one boot). Once all slots hold a dead + * client's directory, a NEW client's registration found no slot and + * every walk of its VAs failed (copies rc=-2, launches reading + * stale pages - the 10th-client mid-boot degradation). Evict the + * least-recently-registered entry instead of refusing the new one; + * the old client is gone and its directory pointer is dead guest + * memory, so the eviction can only remove stale authority. + */ + if (!vs) { + unsigned long oldest = 0; + int oi = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + if (!oldest || time_before(svc.vas[i].stamp, oldest)) { + oldest = svc.vas[i].stamp; + oi = i; + } + } + vs = &svc.vas[oi]; + pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n", + vs->h_vaspace, vs->pdb, desc->h_vaspace); + } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + vs->stamp = jiffies; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } @@ -1932,6 +1967,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, } spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r459: try the NEWEST registration first. A dead client's stale + * directory (the table outlives clients; only 8 slots) can still + * resolve a VA - through guest memory the driver has since reused - + * to a wrong page, and the first success wins. Ordering by the + * registration stamp makes the live client's directory the + * authority and only falls through to older ones when it cannot + * answer. + */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n; i++) { int alt; @@ -5997,14 +6052,18 @@ static void svc_copy_snap_free(u64 dst, u64 src, size_t total, } /* - * r444: drop every snapshot still owned by one entry. A snapshot that was - * not consumed by the entry's own copy execution (the copy ran via the - * forwarded hardware path, or a post-execution re-decode re-captured the - * unchanged staging) has no future consumer once the entry's release is - * delivered - leaving it behind let a LATER entry whose copy shares the - * (dst,src,total) triple execute from the stale bytes. + * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's + * own successful consumption (svc_copy) -> free on ring-slot reuse with a + * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an + * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was + * removed from every opdef_clear caller: an entry whose release was + * delivered can still be REPLAYED (the pin-batch rewind) with its copy + * still owed, and freeing at that point let the replay re-capture the + * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's + * bytes into db1). A stale snapshot can only ever serve its own + * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed. */ -static void svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) +static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) { u32 i; @@ -6032,6 +6091,24 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx, owner_hash)) return; /* already captured */ + /* + * r456: evict any snapshot left by the SAME ring slot under a + * DIFFERENT push hash (the slot was reused by a fresh push). The + * surviving-across-retries snapshot (svc_opdef_clear APPLY no + * longer frees it) must never be readable by the next occupant of + * the slot; the pass hash changes whenever the guest rewrites the + * push, so a same-slot different-hash snapshot is by definition + * stale for the new push. + */ + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash != owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } + } for (i = 0; i < SVC_COPY_SNAP_N; i++) { if (!svc_copy_snaps[i].buf) { s = &svc_copy_snaps[i]; @@ -7452,6 +7529,18 @@ svc_wait_done: */ if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; + /* + * r456: this op already ran in an earlier pass. Any + * staging snapshot a later re-decode pass re-captured + * for it (the capture sites run on every decode) is + * a leak that the r444 APPLY-free used to sweep; with + * that free removed (r456 keeps the snapshot alive + * across retries) drop it here so it can never serve + * a re-push of the ring slot. + */ + if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX) + svc_copy_snap_free(m->dst, m->src, m->bytes, + ch_h, entry_idx, pass_hash); continue; } @@ -9873,6 +9962,23 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; + /* + * r458 THE memset2d root cause: the driver encodes + * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT + * (0x41c) as ONE count=2 method group - observed raw + * push bytes `20028106 00001000 00000100` = header + * (subch 4, count 2, addr 0x418), line length 4096, + * line count 0x100 = 256. Reading only v0 left + * line_count = 0, so the 2D-copy builder below took + * the single-row path: one 4 KB op per push, and a + * 4 MB cudaMemcpy2D delivered exactly one page per + * 1 MB block (memset2d probe: per-MB ok=4096, + * bad=12.5M/12.58M). The GRCE variant below has + * always decoded the count=2 form; the CE path did + * not. + */ + if (count >= 2) + st.line_count = v1; break; /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ case CE_M_LINE_COUNT: @@ -10180,6 +10286,39 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->spitch = spitch; } } + /* + * r458 memset2d forensics: the + * driver's cudaMemcpy2D pushes decode + * LINE_LENGTH=4096 with PITCH_IN/OUT + * present but LINE_COUNT absent + * (lc=0), so only one 4 KB row per + * push executes and a 4 MB copy + * delivers 4 pages. Dump the raw push + * (bounded) for every pitch-shaped + * copy so the method stream that + * carries the row count is on record. + */ + if (gsp_sema_dbg && + (st.have_pitch_out || + st.have_pitch_in) && + svc.op2d_raw_log < 12) { + u32 dl = len < 220 ? len : 220; + + svc.op2d_raw_log++; + pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n", + ch->h_object, push_va, + len, off, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + m->dst, m->src, + (u32)m->bytes, dma); + pr_info("sema-op2d-raw: raw %*phN\n", + dl, buf); + } n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { @@ -15876,10 +16015,29 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) e->n ? e->rel[0].va : 0ull, e->n ? e->rel[0].payload : 0u); } - /* r444: the entry is being consumed; no retry can need its - * staging snapshot any more (see svc_copy_snap_free_owner). */ - if (e->active) - svc_copy_snap_free_owner(e->h_object, e->entry_idx); + /* + * r456/r457: the snapshot lifecycle is now "capture once, consume + * on the copy's success, evict on ring-slot reuse, TTL". Every + * opdef_clear caller can still re-decode the entry: + * - APPLY (0): the pass's own op execution (which consumes the + * snapshot on success) follows the apply; + * - FINISH (1): the entry can be REPLAYED after a consume (the + * pin-batch rewind replays entries while mappings stream in), + * and an owed copy still needs the pipeline-order bytes; + * - FLUSH (2): the op stays skipped this pass but a later replay + * may run it. + * The r457 mode-C evidence: the b1 H2D's snapshot was captured at + * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...), + * FINISH freed it on the entry's consume, the rewind replay + * re-captured at 191608.798838 from the REFILLED staging + * (first4=00000000 3d800000 ... = b2) and the deferred execution + * wrote b2's bytes into db1 (dst16_after=000000000000803d...). + * The r444 leak the old free guarded against is already covered + * by the owner key (a stale snapshot can only ever serve its own + * (ch, idx, hash)), the slot-reuse eviction in + * svc_copy_snap_capture, the ops_done skip free in svc_run_ops + * and the 30 s TTL. + */ memset(e, 0, sizeof(*e)); } @@ -21309,10 +21467,55 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * through to the historical * consume so the channel can * never livelock. + * + * r470: an ALL-ZERO push page is + * the same "not the client's work + * yet" condition - the ring entry + * is written (the previous + * session's or the client's own + * slot reuse) but the push + * CONTENT has not landed, so the + * decode sees no QMD and the + * entry would be consumed as a + * no-op; the end-of-pass counter + * publish then advances the + * guest's completion word while + * the real push was never read, + * and the client drains a + * staging buffer that still holds + * the PREVIOUS copy's bytes + * (streams_events: h_c[2] = c0 + * because the c2 D2H's + * completion was published 4.5 s + * before its copy ran, the ring + * slots having been decoded as + * all-zero pushes). Deliberately + * NARROWER than the CE-shaped + * plausibility verdict: compute + * pushes legitimately fail that + * matcher, so only an entirely + * zero page counts as unwritten. */ + { + bool all_zero = false; + + if (!ch->engine_type && + va && len) { + u32 zi; + + all_zero = true; + for (zi = 0; zi < len; + zi++) + if (pb[zi]) { + all_zero = + false; + break; + } + } if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && - svc_gr_qmd_seen > 0 && + (svc_gr_qmd_seen > 0 || + all_zero) && !entry_replay && !(svc_gr_cand.valid && svc_gr_cand.h_object == @@ -21364,6 +21567,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, (u32)idx, va, len); } + } /* * A CE push that should carry this * channel's next tracking release but |