summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_phys_booter.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_phys_booter.c')
-rw-r--r--kernel/gsp_phys_booter.c178
1 files changed, 141 insertions, 37 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index fb46754..42a0f2a 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -140,6 +140,7 @@ static inline bool gsp_phys_gr_copywit_enabled(void)
return READ_ONCE(gsp_phys_gr_copyout_dump_param);
}
+
/*
* r405 late-write probes (the r405 "partial write" class: the scale launch
* of streams2 wrote 1 of its 32 expected in-place pages and every other
@@ -16472,7 +16473,7 @@ static bool gsp_phys_rmcall_gr_init_emitted;
* stale-QMD execution) landed after its launch was shipped.
*/
static u32 gsp_phys_gr_prev_out_bi;
-static u32 gsp_phys_gr_prev_out_csum;
+static u64 gsp_phys_gr_prev_out_csum;
static bool gsp_phys_gr_prev_out_valid;
/* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged
@@ -17389,6 +17390,54 @@ static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void)
}
/*
+ * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern
+ * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan).
+ * Verifies by content where a lost engine store actually landed: the chunk
+ * is the only bulk memory the operand window PTEs map, so a store that
+ * executed but is absent from the writer's tracked pages must be findable
+ * here (a stale-PTE landing) or nowhere (a dropped store).
+ */
+static int gsp_phys_gr_scan_set(const char *val,
+ const struct kernel_param *kp)
+{
+ struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk();
+ u64 pattern = 0;
+ u32 i, hits = 0;
+
+ if (kstrtoull(val, 0, &pattern))
+ return -EINVAL;
+ if (!c->pages)
+ return -ENODEV;
+ for (i = 0; i < c->pages; i++) {
+ const u8 *p = gsp_phys_chunk_page_va(c, i);
+ u32 off;
+
+ if (!p)
+ continue;
+ for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) {
+ if (get_unaligned_le64(p + off) == pattern) {
+ if (hits < 32)
+ pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n",
+ pattern, i, off,
+ GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 +
+ ((u64)i << 12) + off);
+ hits++;
+ }
+ }
+ }
+ pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n",
+ pattern, hits, c->pages);
+ return 0;
+}
+
+static const struct kernel_param_ops gsp_phys_gr_scan_ops = {
+ .set = gsp_phys_gr_scan_set,
+};
+module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200);
+MODULE_PARM_DESC(gsp_phys_gr_scan,
+ "Scan the GR data-bounce chunk for an 8-byte pattern (hex)");
+
+/*
* B-2: page index of a relocated operand VA inside the engine-visible data
* region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations
* and their MAIL_SLOT bounds. Grown region: VAs from
@@ -17838,7 +17887,7 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req,
*/
static noinline u32
gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req,
- const u32 *csum_in, u32 *n_tracked, u64 *hash_out)
+ const u64 *csum_in, u32 *n_tracked, u64 *hash_out)
{
static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES];
struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk();
@@ -17884,15 +17933,24 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req,
*n_tracked = n_all;
for (j = 0; j < n_all; j++) {
const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]);
- u32 ci, csum = 0;
+ u64 fp;
if (!src)
continue;
- for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
- csum += get_unaligned_le32(src + ci);
- if (csum != csum_in[checked[j]])
+ /*
+ * r452: FNV-1a per page, not a 32-bit additive sum. The
+ * add-sum was PROVABLY blind to a power-of-two in-place
+ * scale: doubling every word of a 4 KB page shifts each
+ * exponent by +1, i.e. +2^23 per word, and 1024 words give
+ * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum
+ * bit-identical, the page classified "unchanged", and the
+ * client read the pre-scale values (the deterministic
+ * streams2/scale class). FNV-1a has no such structure.
+ */
+ fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE);
+ if (fp != csum_in[checked[j]])
changed++;
- hash ^= csum;
+ hash ^= fp;
hash *= 1099511628211ULL;
}
if (hash_out)
@@ -17928,7 +17986,7 @@ static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag)
static noinline u32
gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req,
- const u32 *csum_in, u32 *n_tracked)
+ const u64 *csum_in, u32 *n_tracked)
{
u32 n_all = 0, iter;
u32 changed = 0;
@@ -18082,7 +18140,7 @@ static u32 gsp_phys_gr_idem_n;
static u64 gsp_phys_gr_launch_fp(const u32 *qmd,
const struct gsp_phys_gr_submit *req,
- const u32 *csum_in)
+ const u64 *csum_in)
{
u64 h = 1469598103934665603ULL;
u32 i;
@@ -18160,6 +18218,22 @@ static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src,
}
/*
+ * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero
+ * filter. It is never a change-detection fingerprint any more: doubling every
+ * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0
+ * mod 2^32), which silently classified an in-place x2 scale as "no engine
+ * store" and shipped stale values (the deterministic streams2 class).
+ */
+static inline u32 gsp_phys_gr_page_sum32(const u8 *p)
+{
+ u32 ci, csum = 0;
+
+ for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
+ csum += get_unaligned_le32(p + ci);
+ return csum;
+}
+
+/*
* COPYWIT r226 zero filter: an all-zero page is never a meaningful alias
* partner. A fresh client's fill stages long runs of zero pages
* (uninitialized cudaMalloc buffers), and consecutive zero pages produced
@@ -18205,7 +18279,7 @@ static u32 gsp_phys_gr_in_census_log;
* again). Returns the number of pages staged.
*/
static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
- u32 *csum_in, u64 *staged_hpa)
+ u64 *csum_in, u64 *staged_hpa)
{
struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk();
struct {
@@ -18242,7 +18316,9 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
"gr-bounce-copy-in");
for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
csum += get_unaligned_le32(dst + ci);
- csum_in[bi] = csum;
+ /* r452: the change baseline is the FNV-1a fingerprint (the
+ * add-sum above survives only as the all-zero test). */
+ csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE);
zero = gsp_phys_gr_page_all_zero(csum, dst);
if (!zero && gsp_phys_gr_copywit_enabled()) {
u32 j;
@@ -18291,7 +18367,7 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
* retry, so a retried launch starts from the guest inputs in both sets.
* Returns the number of pages staged.
*/
-static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa)
+static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa)
{
struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk();
u32 gi, copied = 0;
@@ -18299,7 +18375,6 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa)
for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) {
const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa);
u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi);
- u32 ci, csum = 0;
if (!src || !dst)
continue;
@@ -18309,9 +18384,9 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa)
staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa;
gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va,
"gr-bounce-copy-in");
- for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
- csum += get_unaligned_le32(dst + ci);
- csum_in[gsp_phys_gr_gap[gi].bi] = csum;
+ /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */
+ csum_in[gsp_phys_gr_gap[gi].bi] =
+ gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE);
copied++;
}
dma_wmb();
@@ -18907,7 +18982,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
* stack). Safe as static: the only caller is the single-inflight
* ordered gsp-phys-work item.
*/
- static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES];
+ static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES];
/*
* r378: which guest page's content each bounce slot holds this
* launch (set by the copy-in fills, cleared per submission). The
@@ -21441,14 +21516,12 @@ gr_page_unmapped:
const u8 *pp = gsp_phys_chunk_page_va(
gsp_phys_gr_data_chunk(),
gsp_phys_gr_prev_out_bi);
- u32 pc = 0, pi;
+ u64 pc;
if (pp) {
- for (pi = 0; pi < GSP_PHYS_PAGE_SIZE;
- pi += 4)
- pc += get_unaligned_le32(pp + pi);
+ pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE);
if (pc != gsp_phys_gr_prev_out_csum)
- pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u csum %08x -> %08x (late store after its launch shipped)\n",
+ pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n",
gsp_phys_gr_prev_out_bi,
gsp_phys_gr_prev_out_csum,
pc);
@@ -21545,7 +21618,7 @@ gr_page_unmapped:
for (wbi = 0; wbi < req->n_data; wbi++) {
u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va);
const u8 *src;
- u32 ci, csum = 0;
+ u64 fp;
if (bi == GSP_PHYS_GR_DATA_BI_NONE ||
!req->data[wbi].hpa)
@@ -21566,9 +21639,8 @@ gr_page_unmapped:
aliased++;
continue;
}
- for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
- csum += get_unaligned_le32(src + ci);
- if (csum == bounce_csum_in[bi]) {
+ fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE);
+ if (fp == bounce_csum_in[bi]) {
/*
* r378: the engine did not touch this page,
* so the guest page still holds exactly this
@@ -21577,6 +21649,37 @@ gr_page_unmapped:
* stale-content corruption at worst - skip.
*/
unchanged++;
+ /*
+ * r452: witness the UNCHANGED page's chunk
+ * content (first 4 words) and the guest
+ * page's, bounded per launch. This is the
+ * only place the two can be compared for a
+ * page the csum rule classified as "engine
+ * did not touch" - it separates "the store
+ * never landed in the chunk" from "the
+ * store landed but the csum rule missed
+ * it" (or "the reader resolves a different
+ * page than the writer wrote").
+ */
+ if (gsp_phys_gr_copywit_enabled() &&
+ outwit < GSP_PHYS_GR_COPYWIT_MAX) {
+ const u8 *gp = req->data[wbi].hpa ?
+ (const u8 *)__va(req->data[wbi].hpa) :
+ NULL;
+
+ outwit++;
+ pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n",
+ bi, req->data[wbi].va, fp,
+ bounce_csum_in[bi],
+ get_unaligned_le32(src),
+ get_unaligned_le32(src + 4),
+ get_unaligned_le32(src + 8),
+ get_unaligned_le32(src + 12),
+ gp ? get_unaligned_le32(gp) : 0,
+ gp ? get_unaligned_le32(gp + 4) : 0,
+ gp ? get_unaligned_le32(gp + 8) : 0,
+ gp ? get_unaligned_le32(gp + 12) : 0);
+ }
continue;
}
{
@@ -21627,13 +21730,13 @@ gr_page_unmapped:
&rap, &rval);
outwit++;
- pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%08x post=%08x res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n",
+ pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n",
req->data[wbi].va,
req->data[wbi].hpa, bi, f8,
(u64)get_unaligned_le64(
src + GSP_PHYS_PAGE_SIZE -
8),
- bounce_csum_in[bi], csum,
+ bounce_csum_in[bi], fp,
rhave ? "ledger" : "none",
rpa, rap, rval,
(void *)gsp_sema_gr_shadow_base());
@@ -21648,7 +21751,8 @@ gr_page_unmapped:
* evidence).
*/
if (gsp_phys_gr_copywit_enabled() &&
- !gsp_phys_gr_page_all_zero(csum, src)) {
+ !gsp_phys_gr_page_all_zero(
+ gsp_phys_gr_page_sum32(src), src)) {
u32 wi;
for (wi = 0; wi < gsp_phys_gr_outwit_n;
@@ -21684,7 +21788,7 @@ gr_page_unmapped:
*/
if (changed == 0) {
gsp_phys_gr_prev_out_bi = bi;
- gsp_phys_gr_prev_out_csum = csum;
+ gsp_phys_gr_prev_out_csum = fp;
gsp_phys_gr_prev_out_valid = true;
}
changed++;
@@ -21765,8 +21869,7 @@ gr_page_unmapped:
if (bi == GSP_PHYS_GR_DATA_BI_NONE)
continue;
wp = gsp_phys_chunk_page_va(dchunk, bi);
- for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4)
- wc += get_unaligned_le32(wp + k);
+ (void)k; (void)wc;
pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n",
bi,
req->data[wi].reloc_va,
@@ -21774,7 +21877,9 @@ gr_page_unmapped:
get_unaligned_le32(wp + 4),
get_unaligned_le32(wp + 8),
get_unaligned_le32(wp + 12),
- wc != bounce_csum_in[bi]);
+ gsp_phys_gr_fnv1a64(
+ wp, GSP_PHYS_PAGE_SIZE) !=
+ bounce_csum_in[bi]);
}
}
pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n",
@@ -21797,7 +21902,7 @@ gr_page_unmapped:
for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) {
u32 bi = gsp_phys_gr_gap[gi].bi;
const u8 *src = gsp_phys_chunk_page_va(dchunk, bi);
- u32 ci, csum = 0;
+ u64 fp;
if (!src)
continue;
@@ -21810,9 +21915,8 @@ gr_page_unmapped:
gskipped++;
continue;
}
- for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
- csum += get_unaligned_le32(src + ci);
- if (csum == bounce_csum_in[bi]) {
+ fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE);
+ if (fp == bounce_csum_in[bi]) {
/* Engine did not touch it: nothing to ship. */
gunchanged++;
continue;
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-22 18:36:01 +0000