summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c205
-rw-r--r--kernel/gsp_phys_gpu.c62
-rw-r--r--kernel/gsp_phys_gpu.h1
-rw-r--r--kernel/gsp_sema_service.c289
4 files changed, 497 insertions, 60 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 967dc25..42a0f2a 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -140,6 +140,7 @@ static inline bool gsp_phys_gr_copywit_enabled(void)
return READ_ONCE(gsp_phys_gr_copyout_dump_param);
}
+
/*
* r405 late-write probes (the r405 "partial write" class: the scale launch
* of streams2 wrote 1 of its 32 expected in-place pages and every other
@@ -16472,7 +16473,7 @@ static bool gsp_phys_rmcall_gr_init_emitted;
* stale-QMD execution) landed after its launch was shipped.
*/
static u32 gsp_phys_gr_prev_out_bi;
-static u32 gsp_phys_gr_prev_out_csum;
+static u64 gsp_phys_gr_prev_out_csum;
static bool gsp_phys_gr_prev_out_valid;
/* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged
@@ -17389,6 +17390,54 @@ static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void)
}
/*
+ * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern
+ * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan).
+ * Verifies by content where a lost engine store actually landed: the chunk
+ * is the only bulk memory the operand window PTEs map, so a store that
+ * executed but is absent from the writer's tracked pages must be findable
+ * here (a stale-PTE landing) or nowhere (a dropped store).
+ */
+static int gsp_phys_gr_scan_set(const char *val,
+ const struct kernel_param *kp)
+{
+ struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk();
+ u64 pattern = 0;
+ u32 i, hits = 0;
+
+ if (kstrtoull(val, 0, &pattern))
+ return -EINVAL;
+ if (!c->pages)
+ return -ENODEV;
+ for (i = 0; i < c->pages; i++) {
+ const u8 *p = gsp_phys_chunk_page_va(c, i);
+ u32 off;
+
+ if (!p)
+ continue;
+ for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) {
+ if (get_unaligned_le64(p + off) == pattern) {
+ if (hits < 32)
+ pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n",
+ pattern, i, off,
+ GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 +
+ ((u64)i << 12) + off);
+ hits++;
+ }
+ }
+ }
+ pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n",
+ pattern, hits, c->pages);
+ return 0;
+}
+
+static const struct kernel_param_ops gsp_phys_gr_scan_ops = {
+ .set = gsp_phys_gr_scan_set,
+};
+module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200);
+MODULE_PARM_DESC(gsp_phys_gr_scan,
+ "Scan the GR data-bounce chunk for an 8-byte pattern (hex)");
+
+/*
* B-2: page index of a relocated operand VA inside the engine-visible data
* region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations
* and their MAIL_SLOT bounds. Grown region: VAs from
@@ -17838,7 +17887,7 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req,
*/
static noinline u32
gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req,
- const u32 *csum_in, u32 *n_tracked, u64 *hash_out)
+ const u64 *csum_in, u32 *n_tracked, u64 *hash_out)
{
static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES];
struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk();
@@ -17884,15 +17933,24 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req,
*n_tracked = n_all;
for (j = 0; j < n_all; j++) {
const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]);
- u32 ci, csum = 0;
+ u64 fp;
if (!src)
continue;
- for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
- csum += get_unaligned_le32(src + ci);
- if (csum != csum_in[checked[j]])
+ /*
+ * r452: FNV-1a per page, not a 32-bit additive sum. The
+ * add-sum was PROVABLY blind to a power-of-two in-place
+ * scale: doubling every word of a 4 KB page shifts each
+ * exponent by +1, i.e. +2^23 per word, and 1024 words give
+ * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum
+ * bit-identical, the page classified "unchanged", and the
+ * client read the pre-scale values (the deterministic
+ * streams2/scale class). FNV-1a has no such structure.
+ */
+ fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE);
+ if (fp != csum_in[checked[j]])
changed++;
- hash ^= csum;
+ hash ^= fp;
hash *= 1099511628211ULL;
}
if (hash_out)
@@ -17928,7 +17986,7 @@ static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag)
static noinline u32
gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req,
- const u32 *csum_in, u32 *n_tracked)
+ const u64 *csum_in, u32 *n_tracked)
{
u32 n_all = 0, iter;
u32 changed = 0;
@@ -18082,7 +18140,7 @@ static u32 gsp_phys_gr_idem_n;
static u64 gsp_phys_gr_launch_fp(const u32 *qmd,
const struct gsp_phys_gr_submit *req,
- const u32 *csum_in)
+ const u64 *csum_in)
{
u64 h = 1469598103934665603ULL;
u32 i;
@@ -18160,6 +18218,22 @@ static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src,
}
/*
+ * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero
+ * filter. It is never a change-detection fingerprint any more: doubling every
+ * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0
+ * mod 2^32), which silently classified an in-place x2 scale as "no engine
+ * store" and shipped stale values (the deterministic streams2 class).
+ */
+static inline u32 gsp_phys_gr_page_sum32(const u8 *p)
+{
+ u32 ci, csum = 0;
+
+ for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
+ csum += get_unaligned_le32(p + ci);
+ return csum;
+}
+
+/*
* COPYWIT r226 zero filter: an all-zero page is never a meaningful alias
* partner. A fresh client's fill stages long runs of zero pages
* (uninitialized cudaMalloc buffers), and consecutive zero pages produced
@@ -18205,7 +18279,7 @@ static u32 gsp_phys_gr_in_census_log;
* again). Returns the number of pages staged.
*/
static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
- u32 *csum_in, u64 *staged_hpa)
+ u64 *csum_in, u64 *staged_hpa)
{
struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk();
struct {
@@ -18242,7 +18316,9 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
"gr-bounce-copy-in");
for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
csum += get_unaligned_le32(dst + ci);
- csum_in[bi] = csum;
+ /* r452: the change baseline is the FNV-1a fingerprint (the
+ * add-sum above survives only as the all-zero test). */
+ csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE);
zero = gsp_phys_gr_page_all_zero(csum, dst);
if (!zero && gsp_phys_gr_copywit_enabled()) {
u32 j;
@@ -18291,7 +18367,7 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
* retry, so a retried launch starts from the guest inputs in both sets.
* Returns the number of pages staged.
*/
-static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa)
+static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa)
{
struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk();
u32 gi, copied = 0;
@@ -18299,7 +18375,6 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa)
for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) {
const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa);
u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi);
- u32 ci, csum = 0;
if (!src || !dst)
continue;
@@ -18309,9 +18384,9 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa)
staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa;
gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va,
"gr-bounce-copy-in");
- for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
- csum += get_unaligned_le32(dst + ci);
- csum_in[gsp_phys_gr_gap[gi].bi] = csum;
+ /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */
+ csum_in[gsp_phys_gr_gap[gi].bi] =
+ gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE);
copied++;
}
dma_wmb();
@@ -18574,6 +18649,7 @@ static u32 gsp_phys_m91_fault_start;
static u32 gsp_phys_m91_cap_used;
static bool gsp_phys_m91_cap_suppressed;
static bool gsp_phys_m91_icache_log;
+static bool gsp_phys_r449_ccache_log;
static bool gsp_phys_gr_rel_membar_log;
static noinline void
gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req,
@@ -18906,7 +18982,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
* stack). Safe as static: the only caller is the single-inflight
* ordered gsp-phys-work item.
*/
- static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES];
+ static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES];
/*
* r378: which guest page's content each bounce slot holds this
* launch (set by the copy-in fills, cleared per submission). The
@@ -19084,6 +19160,32 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
}
}
/*
+ * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default
+ * 0 = off): the same cache-control word's bit 31 (the V01_06-layout
+ * SHADER_CONSTANT_CACHE position) is never set by the captured guest
+ * QMDs. The forwarded param page is rewritten in place between launches
+ * (the two-bank win0 pool), so a stale constant-cache line can serve
+ * the PREVIOUS launch's parameters and the current launch's output
+ * pages stay untouched - the post-dispatch no-store signature. Mode 1
+ * sets bit 31; mode 2 forces the whole invalidate set (bits 24..31).
+ * Only when the QMD carries the driver's standard cache-control
+ * signature (bits 26..29) so an unknown QMD revision is never touched.
+ */
+ if (gsp_phys_gr_ccache_inv_value() &&
+ (qmd[5] & 0x3c000000u) == 0x3c000000u) {
+ u32 mode = gsp_phys_gr_ccache_inv_value();
+
+ if (mode == 2)
+ qmd[5] |= 0xff000000u;
+ else
+ qmd[5] |= 1u << 31;
+ if (!gsp_phys_r449_ccache_log) {
+ gsp_phys_r449_ccache_log = true;
+ pr_info("vgsp-gr: r449 constant-cache invalidate mode=%u applied (w5=0x%08x)\n",
+ mode, qmd[5]);
+ }
+ }
+ /*
* r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar): force the
* forwarded QMD's RELEASE_MEMBAR_TYPE to FE_SYSMEMBAR (QMDV00_06
* word 11 bit 14) so the front-end flushes its pending system-memory
@@ -21414,14 +21516,12 @@ gr_page_unmapped:
const u8 *pp = gsp_phys_chunk_page_va(
gsp_phys_gr_data_chunk(),
gsp_phys_gr_prev_out_bi);
- u32 pc = 0, pi;
+ u64 pc;
if (pp) {
- for (pi = 0; pi < GSP_PHYS_PAGE_SIZE;
- pi += 4)
- pc += get_unaligned_le32(pp + pi);
+ pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE);
if (pc != gsp_phys_gr_prev_out_csum)
- pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u csum %08x -> %08x (late store after its launch shipped)\n",
+ pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n",
gsp_phys_gr_prev_out_bi,
gsp_phys_gr_prev_out_csum,
pc);
@@ -21518,7 +21618,7 @@ gr_page_unmapped:
for (wbi = 0; wbi < req->n_data; wbi++) {
u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va);
const u8 *src;
- u32 ci, csum = 0;
+ u64 fp;
if (bi == GSP_PHYS_GR_DATA_BI_NONE ||
!req->data[wbi].hpa)
@@ -21539,9 +21639,8 @@ gr_page_unmapped:
aliased++;
continue;
}
- for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
- csum += get_unaligned_le32(src + ci);
- if (csum == bounce_csum_in[bi]) {
+ fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE);
+ if (fp == bounce_csum_in[bi]) {
/*
* r378: the engine did not touch this page,
* so the guest page still holds exactly this
@@ -21550,6 +21649,37 @@ gr_page_unmapped:
* stale-content corruption at worst - skip.
*/
unchanged++;
+ /*
+ * r452: witness the UNCHANGED page's chunk
+ * content (first 4 words) and the guest
+ * page's, bounded per launch. This is the
+ * only place the two can be compared for a
+ * page the csum rule classified as "engine
+ * did not touch" - it separates "the store
+ * never landed in the chunk" from "the
+ * store landed but the csum rule missed
+ * it" (or "the reader resolves a different
+ * page than the writer wrote").
+ */
+ if (gsp_phys_gr_copywit_enabled() &&
+ outwit < GSP_PHYS_GR_COPYWIT_MAX) {
+ const u8 *gp = req->data[wbi].hpa ?
+ (const u8 *)__va(req->data[wbi].hpa) :
+ NULL;
+
+ outwit++;
+ pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n",
+ bi, req->data[wbi].va, fp,
+ bounce_csum_in[bi],
+ get_unaligned_le32(src),
+ get_unaligned_le32(src + 4),
+ get_unaligned_le32(src + 8),
+ get_unaligned_le32(src + 12),
+ gp ? get_unaligned_le32(gp) : 0,
+ gp ? get_unaligned_le32(gp + 4) : 0,
+ gp ? get_unaligned_le32(gp + 8) : 0,
+ gp ? get_unaligned_le32(gp + 12) : 0);
+ }
continue;
}
{
@@ -21600,13 +21730,13 @@ gr_page_unmapped:
&rap, &rval);
outwit++;
- pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%08x post=%08x res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n",
+ pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n",
req->data[wbi].va,
req->data[wbi].hpa, bi, f8,
(u64)get_unaligned_le64(
src + GSP_PHYS_PAGE_SIZE -
8),
- bounce_csum_in[bi], csum,
+ bounce_csum_in[bi], fp,
rhave ? "ledger" : "none",
rpa, rap, rval,
(void *)gsp_sema_gr_shadow_base());
@@ -21621,7 +21751,8 @@ gr_page_unmapped:
* evidence).
*/
if (gsp_phys_gr_copywit_enabled() &&
- !gsp_phys_gr_page_all_zero(csum, src)) {
+ !gsp_phys_gr_page_all_zero(
+ gsp_phys_gr_page_sum32(src), src)) {
u32 wi;
for (wi = 0; wi < gsp_phys_gr_outwit_n;
@@ -21657,7 +21788,7 @@ gr_page_unmapped:
*/
if (changed == 0) {
gsp_phys_gr_prev_out_bi = bi;
- gsp_phys_gr_prev_out_csum = csum;
+ gsp_phys_gr_prev_out_csum = fp;
gsp_phys_gr_prev_out_valid = true;
}
changed++;
@@ -21738,8 +21869,7 @@ gr_page_unmapped:
if (bi == GSP_PHYS_GR_DATA_BI_NONE)
continue;
wp = gsp_phys_chunk_page_va(dchunk, bi);
- for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4)
- wc += get_unaligned_le32(wp + k);
+ (void)k; (void)wc;
pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n",
bi,
req->data[wi].reloc_va,
@@ -21747,7 +21877,9 @@ gr_page_unmapped:
get_unaligned_le32(wp + 4),
get_unaligned_le32(wp + 8),
get_unaligned_le32(wp + 12),
- wc != bounce_csum_in[bi]);
+ gsp_phys_gr_fnv1a64(
+ wp, GSP_PHYS_PAGE_SIZE) !=
+ bounce_csum_in[bi]);
}
}
pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n",
@@ -21770,7 +21902,7 @@ gr_page_unmapped:
for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) {
u32 bi = gsp_phys_gr_gap[gi].bi;
const u8 *src = gsp_phys_chunk_page_va(dchunk, bi);
- u32 ci, csum = 0;
+ u64 fp;
if (!src)
continue;
@@ -21783,9 +21915,8 @@ gr_page_unmapped:
gskipped++;
continue;
}
- for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
- csum += get_unaligned_le32(src + ci);
- if (csum == bounce_csum_in[bi]) {
+ fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE);
+ if (fp == bounce_csum_in[bi]) {
/* Engine did not touch it: nothing to ship. */
gunchanged++;
continue;
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index b61929e..74062dd 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -734,20 +734,30 @@ int gsp_phys_rmcall_gr_qmdwin_value(void)
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value);
/*
- * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod; default 0).
- * When set, a forwarded GR launch ignores the resolved guest release page and
- * points QMD release0 at the module window-0 semaphore VA
- * (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host VA.
- * Discriminant for "engine completed but the shadow release write was lost"
- * vs "engine never completed": with the module release, a completed QMD is
- * always observable. Also makes the existing module-release fallback
- * observable (it previously set no poll source, so it could never complete).
- */
-static int gsp_phys_rmcall_gr_rel_mod_param;
+ * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod).
+ *
+ * DEFAULT 1 since r453. When set, a forwarded GR launch ignores the resolved
+ * guest release page and points QMD release0 at the module window-0 semaphore
+ * VA (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host
+ * VA; the GUEST-VISIBLE release word is authored by the host only on the
+ * completion edge - after the retire wait and after the bounce copy-out has
+ * shipped the engine's stores to the guest pages. Evidence (r453, the FNV
+ * change-detection fix on top):
+ * - with 0: the guest's release could be satisfied by the engine's own
+ * early write while the host copy-out was still pending, so a client's
+ * stream-ordered readback (streams2's c2, memcpy_roundtrip's memset
+ * phase, streams_events' chain) read PRE-launch content: 11/15 battery.
+ * - with 1: 13/15 battery (streams_events and memcpy_roundtrip flipped
+ * green), no stalls across the full battery, mail+saxpy clean.
+ * The r414 stall hole (a launch that never reached the hold edge left the
+ * guest word unauthored) is closed by the r420b per-outcome authoring path;
+ * one-line rollback: echo 0 > .../parameters/gsp_phys_rmcall_gr_rel_mod.
+ */
+static int gsp_phys_rmcall_gr_rel_mod_param = 1;
module_param_named(gsp_phys_rmcall_gr_rel_mod,
gsp_phys_rmcall_gr_rel_mod_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_rel_mod,
- "Force the forwarded GR QMD release0 to the module semaphore (default 0; 1 = m64 engine-completion discriminant)");
+ "Force the forwarded GR QMD release0 to the module semaphore; guest release authored only on the post-copy-out completion edge (default 1 since r453; 0 = historical engine-authored guest release)");
int gsp_phys_rmcall_gr_rel_mod_value(void)
{
@@ -1198,6 +1208,36 @@ int gsp_phys_gr_wedge_rearm_value(void)
EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_rearm_value);
/*
+ * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default 0).
+ * The forwarded QMD's cache-control block (QMDV02_03 word 5) carries the
+ * per-cache invalidate bits; the captured guest QMDs always set bits 26..29
+ * (texture header/sampler/data + shader DATA) and the M91 fix adds bit 30
+ * (INSTRUCTION), but bit 31 (SHADER CONSTANT - where a kernel's launch
+ * parameters live) is never set. The forwarded launch's param page is
+ * rewritten in place between launches (the two-bank win0 pool), so a stale
+ * constant-cache line can serve the PREVIOUS launch's parameters - the
+ * kernel then computes with the previous launch's pointers/sizes and the
+ * current launch's tracked output pages stay untouched ("no stores").
+ * Mode: 0 = off, 1 = set bit 31 only, 2 = force the full 0xff000000
+ * invalidate set. Read at submission time, so it can be A/B'd at runtime.
+ */
+static int gsp_phys_gr_ccache_inv_param;
+module_param_named(gsp_phys_gr_ccache_inv, gsp_phys_gr_ccache_inv_param,
+ int, 0644);
+MODULE_PARM_DESC(gsp_phys_gr_ccache_inv,
+ "r449: forwarded-QMD shader-constant-cache invalidate: 0=off, 1=bit31, 2=all invalidate bits (default 0)");
+
+unsigned int gsp_phys_gr_ccache_inv_value(void)
+{
+ if (gsp_phys_gr_ccache_inv_param < 0)
+ return 0;
+ if (gsp_phys_gr_ccache_inv_param > 2)
+ return 2;
+ return (unsigned int)gsp_phys_gr_ccache_inv_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_gr_ccache_inv_value);
+
+/*
* r410 wedge quiet window (gsp_phys_gr_wedge_quiet_ms; default 40000, clamped
* [0,120000]; 0 disables). The sticky per-lane no-execute wedge (100% of
* launches no-write while kept busy) is proven to CLEAR when the lane stays
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 46d841d..36c2220 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -403,6 +403,7 @@ int gsp_phys_rmcall_gr_cwd_slot_value(void);
* attempt counter (default 3, clamped [0,4]). */
int gsp_phys_gr_nowrite_retries_value(void);
int gsp_phys_gr_wedge_rearm_value(void);
+unsigned int gsp_phys_gr_ccache_inv_value(void);
/* STALLFREE: absolute per-launch GR submit wall-clock budget in ms, clamped
* [10000,600000]; 0 = only the built-in per-ladder bounds (default 60000).
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b90c948..ea96595 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -420,6 +420,18 @@ struct svc_channel {
u64 gr_nocap_va; /* push VA the window was opened for */
unsigned long gr_nocap_since; /* absolute first-seen stamp */
/*
+ * r471: the completion-counter publish must never advance the
+ * guest's word past a push whose content was still unwritten when
+ * the entry was consumed (the streams_events staging-reuse
+ * corruption: the publish satisfied the client's D2H wait before
+ * its copy ran, and the readback drained a staging buffer that
+ * still held the PREVIOUS copy's bytes). Lowest ring slot consumed
+ * with all-zero content; 0 = none. Cleared when a later decode of
+ * a GR push reads non-zero content (the content landed - the rewinds
+ * re-read entries constantly, so this heals within a pass or two).
+ */
+ u32 gr_bad_put;
+ /*
* STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A
* channel that cannot make forward progress (last_get pinned behind
* put, or a residual session put that never drops) is detected here
@@ -486,6 +498,12 @@ struct svc_vaspace {
u64 pdb;
u32 entries; /* root entry count */
int ap; /* SVC_APERTURE_* of the directory itself */
+ unsigned long stamp; /* r459: last registration/use; LRU evict
+ * when the table is full (a stale entry from
+ * a dead client otherwise blocks every new
+ * client after the 8th registration: the
+ * mid-boot walk failure that made the 10th+
+ * client's copies fail rc=-2) */
};
struct svc_lat {
@@ -819,6 +837,7 @@ struct sema_svc {
u64 pushtrace_all; /* fail4: de-windowed push decode
* trace (every push, bounded) */
u64 opadd_log; /* fail4: every op appended */
+ u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */
u64 dmatrace_log; /* fail4: denied-transfer trace */
u64 pend_dbg_log; /* fail4 r305: pending/hold witness */
u64 snap_gate_log; /* fail4 r311: capture-gate witness */
@@ -1599,12 +1618,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
if (!vs && !svc.vas[i].valid)
vs = &svc.vas[i];
}
+ /*
+ * r459: the table is 8 slots and the guest driver does not always
+ * send a teardown for a client's VA space (observed: 23 distinct
+ * registrations, 0 drops in one boot). Once all slots hold a dead
+ * client's directory, a NEW client's registration found no slot and
+ * every walk of its VAs failed (copies rc=-2, launches reading
+ * stale pages - the 10th-client mid-boot degradation). Evict the
+ * least-recently-registered entry instead of refusing the new one;
+ * the old client is gone and its directory pointer is dead guest
+ * memory, so the eviction can only remove stale authority.
+ */
+ if (!vs) {
+ unsigned long oldest = 0;
+ int oi = 0;
+
+ for (i = 0; i < SVC_MAX_VASPACES; i++) {
+ if (!svc.vas[i].valid)
+ continue;
+ if (!oldest || time_before(svc.vas[i].stamp, oldest)) {
+ oldest = svc.vas[i].stamp;
+ oi = i;
+ }
+ }
+ vs = &svc.vas[oi];
+ pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n",
+ vs->h_vaspace, vs->pdb, desc->h_vaspace);
+ }
if (vs) {
vs->valid = true;
vs->h_vaspace = desc->h_vaspace;
vs->pdb = desc->pdb;
vs->entries = desc->num_entries;
vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
+ vs->stamp = jiffies;
/* A new directory invalidates what we learned from the old one. */
memset(svc.lat, 0, sizeof(svc.lat));
}
@@ -1932,6 +1979,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va,
}
spin_unlock_irqrestore(&svc.vas_lock, flags);
+ /*
+ * r459: try the NEWEST registration first. A dead client's stale
+ * directory (the table outlives clients; only 8 slots) can still
+ * resolve a VA - through guest memory the driver has since reused -
+ * to a wrong page, and the first success wins. Ordering by the
+ * registration stamp makes the live client's directory the
+ * authority and only falls through to older ones when it cannot
+ * answer.
+ */
+ for (i = 1; i < n; i++) {
+ struct svc_vaspace k = local[i];
+ int j = i - 1;
+
+ while (j >= 0 && time_before(local[j].stamp, k.stamp)) {
+ local[j + 1] = local[j];
+ j--;
+ }
+ local[j + 1] = k;
+ }
+
for (i = 0; i < n; i++) {
int alt;
@@ -5997,14 +6064,18 @@ static void svc_copy_snap_free(u64 dst, u64 src, size_t total,
}
/*
- * r444: drop every snapshot still owned by one entry. A snapshot that was
- * not consumed by the entry's own copy execution (the copy ran via the
- * forwarded hardware path, or a post-execution re-decode re-captured the
- * unchanged staging) has no future consumer once the entry's release is
- * delivered - leaving it behind let a LATER entry whose copy shares the
- * (dst,src,total) triple execute from the stale bytes.
+ * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's
+ * own successful consumption (svc_copy) -> free on ring-slot reuse with a
+ * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an
+ * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was
+ * removed from every opdef_clear caller: an entry whose release was
+ * delivered can still be REPLAYED (the pin-batch rewind) with its copy
+ * still owed, and freeing at that point let the replay re-capture the
+ * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's
+ * bytes into db1). A stale snapshot can only ever serve its own
+ * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed.
*/
-static void svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx)
+static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx)
{
u32 i;
@@ -6032,6 +6103,24 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst,
if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx,
owner_hash))
return; /* already captured */
+ /*
+ * r456: evict any snapshot left by the SAME ring slot under a
+ * DIFFERENT push hash (the slot was reused by a fresh push). The
+ * surviving-across-retries snapshot (svc_opdef_clear APPLY no
+ * longer frees it) must never be readable by the next occupant of
+ * the slot; the pass hash changes whenever the guest rewrites the
+ * push, so a same-slot different-hash snapshot is by definition
+ * stale for the new push.
+ */
+ for (i = 0; i < SVC_COPY_SNAP_N; i++) {
+ if (svc_copy_snaps[i].buf &&
+ svc_copy_snaps[i].owner_ch == owner_ch &&
+ svc_copy_snaps[i].owner_idx == owner_idx &&
+ svc_copy_snaps[i].owner_hash != owner_hash) {
+ kfree(svc_copy_snaps[i].buf);
+ svc_copy_snaps[i].buf = NULL;
+ }
+ }
for (i = 0; i < SVC_COPY_SNAP_N; i++) {
if (!svc_copy_snaps[i].buf) {
s = &svc_copy_snaps[i];
@@ -7452,6 +7541,18 @@ svc_wait_done:
*/
if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) {
svc.ops_once_skipped++;
+ /*
+ * r456: this op already ran in an earlier pass. Any
+ * staging snapshot a later re-decode pass re-captured
+ * for it (the capture sites run on every decode) is
+ * a leak that the r444 APPLY-free used to sweep; with
+ * that free removed (r456 keeps the snapshot alive
+ * across retries) drop it here so it can never serve
+ * a re-push of the ring slot.
+ */
+ if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX)
+ svc_copy_snap_free(m->dst, m->src, m->bytes,
+ ch_h, entry_idx, pass_hash);
continue;
}
@@ -9873,6 +9974,23 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
break;
case CE_M_LINE_LENGTH_IN:
st.line_length = v0;
+ /*
+ * r458 THE memset2d root cause: the driver encodes
+ * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT
+ * (0x41c) as ONE count=2 method group - observed raw
+ * push bytes `20028106 00001000 00000100` = header
+ * (subch 4, count 2, addr 0x418), line length 4096,
+ * line count 0x100 = 256. Reading only v0 left
+ * line_count = 0, so the 2D-copy builder below took
+ * the single-row path: one 4 KB op per push, and a
+ * 4 MB cudaMemcpy2D delivered exactly one page per
+ * 1 MB block (memset2d probe: per-MB ok=4096,
+ * bad=12.5M/12.58M). The GRCE variant below has
+ * always decoded the count=2 form; the CE path did
+ * not.
+ */
+ if (count >= 2)
+ st.line_count = v1;
break;
/* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */
case CE_M_LINE_COUNT:
@@ -10180,6 +10298,39 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
m->spitch = spitch;
}
}
+ /*
+ * r458 memset2d forensics: the
+ * driver's cudaMemcpy2D pushes decode
+ * LINE_LENGTH=4096 with PITCH_IN/OUT
+ * present but LINE_COUNT absent
+ * (lc=0), so only one 4 KB row per
+ * push executes and a 4 MB copy
+ * delivers 4 pages. Dump the raw push
+ * (bounded) for every pitch-shaped
+ * copy so the method stream that
+ * carries the row count is on record.
+ */
+ if (gsp_sema_dbg &&
+ (st.have_pitch_out ||
+ st.have_pitch_in) &&
+ svc.op2d_raw_log < 12) {
+ u32 dl = len < 220 ? len : 220;
+
+ svc.op2d_raw_log++;
+ pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n",
+ ch->h_object, push_va,
+ len, off,
+ st.line_length,
+ st.line_count,
+ st.have_pitch_out,
+ st.pitch_out,
+ st.have_pitch_in,
+ st.pitch_in,
+ m->dst, m->src,
+ (u32)m->bytes, dma);
+ pr_info("sema-op2d-raw: raw %*phN\n",
+ dl, buf);
+ }
n_ops++;
ops_seq++;
} else if (gsp_sema_dbg && svc.dma_skip_log < 32) {
@@ -14958,6 +15109,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
return;
val = put;
+ /*
+ * r471: never advance the guest's completion word past a push whose
+ * content was still unwritten when its entry was consumed. The
+ * clamp position is the lowest all-zero GR slot (gr_bad_put = slot
+ * + 1; 0 = none). Without it the publish can satisfy a D2H wait
+ * before its copy ran and the client drains a reused staging buffer
+ * holding the previous copy's bytes (streams_events h_c[2] = c0).
+ * The clamp heals when the rewinds re-read the slot with real
+ * content; the real completions still arrive through the decoded
+ * releases.
+ */
+ if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) {
+ val = ch->gr_bad_put - 1;
+ if (gsp_sema_dbg && svc.counter_pub_dbg < 96) {
+ svc.counter_pub_dbg++;
+ pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n",
+ ch->h_object, pa, val, ch->gr_bad_put, cur);
+ }
+ }
if (val == ch->counter_last)
return;
/*
@@ -15876,10 +16046,29 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller)
e->n ? e->rel[0].va : 0ull,
e->n ? e->rel[0].payload : 0u);
}
- /* r444: the entry is being consumed; no retry can need its
- * staging snapshot any more (see svc_copy_snap_free_owner). */
- if (e->active)
- svc_copy_snap_free_owner(e->h_object, e->entry_idx);
+ /*
+ * r456/r457: the snapshot lifecycle is now "capture once, consume
+ * on the copy's success, evict on ring-slot reuse, TTL". Every
+ * opdef_clear caller can still re-decode the entry:
+ * - APPLY (0): the pass's own op execution (which consumes the
+ * snapshot on success) follows the apply;
+ * - FINISH (1): the entry can be REPLAYED after a consume (the
+ * pin-batch rewind replays entries while mappings stream in),
+ * and an owed copy still needs the pipeline-order bytes;
+ * - FLUSH (2): the op stays skipped this pass but a later replay
+ * may run it.
+ * The r457 mode-C evidence: the b1 H2D's snapshot was captured at
+ * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...),
+ * FINISH freed it on the entry's consume, the rewind replay
+ * re-captured at 191608.798838 from the REFILLED staging
+ * (first4=00000000 3d800000 ... = b2) and the deferred execution
+ * wrote b2's bytes into db1 (dst16_after=000000000000803d...).
+ * The r444 leak the old free guarded against is already covered
+ * by the owner key (a stale snapshot can only ever serve its own
+ * (ch, idx, hash)), the slot-reuse eviction in
+ * svc_copy_snap_capture, the ops_done skip free in svc_run_ops
+ * and the 30 s TTL.
+ */
memset(e, 0, sizeof(*e));
}
@@ -21309,10 +21498,81 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
* through to the historical
* consume so the channel can
* never livelock.
+ *
+ * r470: an ALL-ZERO push page is
+ * the same "not the client's work
+ * yet" condition - the ring entry
+ * is written (the previous
+ * session's or the client's own
+ * slot reuse) but the push
+ * CONTENT has not landed, so the
+ * decode sees no QMD and the
+ * entry would be consumed as a
+ * no-op; the end-of-pass counter
+ * publish then advances the
+ * guest's completion word while
+ * the real push was never read,
+ * and the client drains a
+ * staging buffer that still holds
+ * the PREVIOUS copy's bytes
+ * (streams_events: h_c[2] = c0
+ * because the c2 D2H's
+ * completion was published 4.5 s
+ * before its copy ran, the ring
+ * slots having been decoded as
+ * all-zero pushes). Deliberately
+ * NARROWER than the CE-shaped
+ * plausibility verdict: compute
+ * pushes legitimately fail that
+ * matcher, so only an entirely
+ * zero page counts as unwritten.
*/
+ {
+ bool all_zero = false;
+
+ if (!ch->engine_type &&
+ va && len) {
+ u32 zi;
+
+ all_zero = true;
+ for (zi = 0; zi < len;
+ zi++)
+ if (pb[zi]) {
+ all_zero =
+ false;
+ break;
+ }
+ }
+ /*
+ * r471: publish clamp bookkeeping.
+ * A zero-content GR entry that is
+ * held or consumed must never let
+ * the end-of-pass counter publish
+ * advance the guest's completion
+ * word past it (the client would
+ * satisfy its readback wait before
+ * the real push/copy ran - the
+ * staging buffer still holds the
+ * previous copy's bytes). Record
+ * the lowest zero slot; a decode
+ * that reads non-zero content
+ * clears the clamp.
+ */
+ if (!ch->engine_type) {
+ if (all_zero) {
+ if (!ch->gr_bad_put ||
+ (u32)idx + 1 <
+ ch->gr_bad_put)
+ ch->gr_bad_put =
+ (u32)idx + 1;
+ } else if (va && len) {
+ ch->gr_bad_put = 0;
+ }
+ }
if (!ch->engine_type && va && len &&
gsp_phys_gr_forward_enabled() &&
- svc_gr_qmd_seen > 0 &&
+ (svc_gr_qmd_seen > 0 ||
+ all_zero) &&
!entry_replay &&
!(svc_gr_cand.valid &&
svc_gr_cand.h_object ==
@@ -21364,6 +21624,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
ch->h_object,
(u32)idx, va, len);
}
+ }
/*
* A CE push that should carry this
* channel's next tracking release but
@@ -22099,6 +22360,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc.chan[k].push_hold_until = ch->push_hold_until;
svc.chan[k].defer_since = ch->defer_since;
svc.chan[k].put_prev = ch->put_prev;
+ /* r471: the publish clamp watermark is set
+ * and cleared on the pump thread; persist it
+ * so the next pass still clamps. */
+ svc.chan[k].gr_bad_put = ch->gr_bad_put;
/*
* Persist the cheap-wait window and the content
* watermark: the pump works on a snapshot, so
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-22 18:30:42 +0000