From ccf57b47b2c68ede8c0602b13070bd4788cad8c8 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sat, 31 Oct 2026 11:26:00 -0400 Subject: vgsp: svc_lat_drop + trust sub-source tag (sub= in sema-pushdec) + SVC_ZRW zero-streak table (src>=2 all-zero x2 -> drop lat/re-walk, pin when src==2), gl_win_start arms only on hdr!=0 (root cause of 5-boot capture drought: window armed on zero no-op), svc_gl_flush zero-lead advance, svc_push_pa_ceiling from gsp_phys_guest_ram_top - capture drought BROKEN (r1131 win=1429 words, 1007 section captures, 0 zero-head refusals); MME-PRELUDE ships in one launch FIRST TIME (gen=4 words=901 + window=390 -> GR LAUNCH words=1348); stores still 0 (engine fetch NONE, MISSING_MACRO_DATA ESR 0x80000001; 8.8s fallback); next: one-variable attribution (sa_stub=0 then surf_prelude=0) + same-generation SA/program capture - SESSION 153 r1129-r1131 --- kernel/gsp_phys_gpu.c | 35 ++++++++ kernel/gsp_phys_gpu.h | 9 ++ kernel/gsp_sema_service.c | 208 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 243 insertions(+), 9 deletions(-) diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index e9c0c5a..f1d72f7 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1953,6 +1953,41 @@ bool gsp_phys_guest_ram_active(void) } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); +/* + * Highest GPA any registered guest-RAM range covers. Conservative global + * bound (max across the legacy and every per-mdev record) for consumers that + * only need to reject translations pointing past RAM - e.g. the sema push + * read's PA ceiling (r1127: a stale VAS walk answered 0x23921d000 = 8.89 GB + * for the beef setup push; with the 8/10 GB guest layout the answer is + * inside the registered ranges and reads padding zeros, so no bound alone + * fixes it - the repeated-zero invalidation does - but any answer ABOVE + * this top can never read content). 0 when nothing is registered. + */ +u64 gsp_phys_guest_ram_top(void) +{ + u64 top = 0; + unsigned int i, j; + + mutex_lock(&guest_ram_lock); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS + 1; i++) { + const struct gsp_phys_guest_ram_rec *rec = + (i == 0) ? &guest_ram_legacy : &guest_ram_mdevs[i - 1]; + + if (!rec->file) + continue; + for (j = 0; j < rec->ranges_n; j++) { + u64 end = rec->ranges[j].gpa_base + + rec->ranges[j].size; + + if (end > top) + top = end; + } + } + mutex_unlock(&guest_ram_lock); + return top; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_top); + /* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, * path, range count and every range (gpa/offset/size). With only the legacy diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 3993c6e..4d9f47b 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -515,6 +515,15 @@ void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu); bool gsp_phys_guest_ram_active(void); void gsp_phys_guest_ram_state(char *buf, size_t len); +/* + * Highest GPA any registered guest-RAM range covers (max of gpa_base+size + * across every active record; 0 when nothing is registered). A GPA at or + * above this can never hold guest content - the backing file's padding + * reads as zeros - so translation consumers use it to reject walk/pin + * answers that point past the registered RAM. + */ +u64 gsp_phys_guest_ram_top(void); + /* * Open-mdev helpers for the per-mdev debugfs selector and the single-guest * guards: count the mdevs whose VFIO device is open, or resolve one open diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 2b435c4..748eae9 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3374,6 +3374,27 @@ static void svc_lat_store(u64 va, u64 addr, int ap) l->valid = true; } +/* + * SESSION 153-CONTINUED-67: invalidate a learned-cache entry so the next + * lookup of this page re-runs the source chain underneath it (discovered + * map, registered-VAS walk, discovered walk, targeted resolve) instead of + * reusing a cached answer. Used by the repeated all-zero read + * invalidation in svc_read_va_push_ex: every walk success re-stores the + * answer here (svc_lat_store after each svc_lookup_trusted sub-source), + * so a stale walk result otherwise shadows a fresh walk for the rest of + * the boot (r1127: the beef setup push kept answering the same zero page + * while the client's content was live elsewhere). + */ +static void svc_lat_drop(u64 va) +{ + struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; + + if (l->valid && + (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (va & ~(u64)(SEMA_PAGE_SIZE - 1))) + l->valid = false; +} + /* * r820: driver-declared kernel-buffer translations (PROMOTE_CTX). * @@ -4293,11 +4314,22 @@ static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, return -ENOENT; } +/* + * SESSION 153-CONTINUED-67: which sub-source of svc_lookup_trusted answered + * the last call (1=channel VAS 2=pin 3=learned 4=discovered map 5=VAS walk + * 6=discovered walk 7=targeted 8=promote 0=miss). Printed as `sub=` by the + * pushdec witness so a repeated-zero read can be attributed to the exact + * layer (the r1127 investigation could not tell a stale learned entry from + * a stale registered-VAS walk). + */ +static u8 svc_trust_src; + static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { u64 pa = 0; + svc_trust_src = 0; /* * r835: the decoded channel's own page tables answer first. When a * channel is being serviced (svc_cur_vas set) its VAS is the only @@ -4305,27 +4337,35 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, * hold another client's pages for the same numeric VA. A failed * walk falls through to the old chain unchanged. */ - if (!svc_chanvas_lookup(vgpu, va, addr, ap)) + if (!svc_chanvas_lookup(vgpu, va, addr, ap)) { + svc_trust_src = 1; return 0; + } if (!svc_pin_lookup_any(va, addr, ap)) { + svc_trust_src = 2; return 0; } - if (!svc_lat_lookup(va, addr, ap)) + if (!svc_lat_lookup(va, addr, ap)) { + svc_trust_src = 3; return 0; + } if (!svc_map_lookup(va, &pa)) { *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); + svc_trust_src = 4; return 0; } if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 5; return 0; } if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 6; return 0; } if (!svc_pd_found) { @@ -4340,6 +4380,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 6; return 0; } } @@ -4348,6 +4389,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); + svc_trust_src = 7; return 0; } /* @@ -4365,6 +4407,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, */ if (!svc_promo_lookup(va, addr, ap)) { svc_lat_store(va, *addr, *ap); + svc_trust_src = 8; return 0; } return -ENOENT; @@ -9390,6 +9433,21 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * when the window head is a real word - a * later pass/entry on the filled page captures * the frame. + * + * SESSION 153-CONTINUED-67: the w2/r1127 + * zero-head refusals were NOT mid-write + * windows that later filled: the same word at + * the same offset read zero across every pass + * of the boot (64 refusals, census content + * stable at 0x1f8..0x614), because the window + * start had been armed on a zero word (a + * no-op INC(0) slot) in the gap after the last + * foreign-subchannel run. A zero word is not a + * method: advance past the leading zero words + * to the first real word and capture the + * window from there. Only a window that is + * ENTIRELY zero is still refused (the r1031 + * protection). */ if (at_off > svc_gl.start_off && svc_gl.start_off + 4u <= len) { @@ -9397,16 +9455,39 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, memcpy(&w0h, buf + svc_gl.start_off, 4); if (!w0h) { - if (gsp_sema_dbg && - svc_gl_np_log < 64) { + u32 adv = svc_gl.start_off; + u32 lim = min_t(u32, at_off, + len); + + while (adv + 4u <= lim) { + memcpy(&w0h, + buf + adv, 4); + if (w0h) + break; + adv += 4u; + } + if (!w0h) { + if (gsp_sema_dbg && + svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (whole window zero; not captured)\n", + ch->h_object, + push_va, + svc_gl.start_off, + (at_off - svc_gl.start_off) / 4u); + } + goto out; + } + if (svc_gl_np_log < 64) { svc_gl_np_log++; - pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (page mid-write; not captured)\n", + pr_info("sema-gl: zero-lead window h=0x%x push=0x%llx start=0x%x->0x%x win=%u words (skipped zero no-ops; capturing)\n", ch->h_object, push_va, svc_gl.start_off, - (at_off - svc_gl.start_off) / 4u); + adv, + (at_off - adv) / 4u); } - goto out; + svc_gl.start_off = adv; } } /* @@ -15581,7 +15662,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (as != 0xffu) { if (subch != as) gl_win_start = 0; - else if (!gl_win_start && + else if (!gl_win_start && hdr && !(op == 1 && addr >= 0x1b00 && addr <= 0x1b0c)) gl_win_start = off; @@ -19844,11 +19925,23 @@ static u64 svc_lastgood_ring_pa; static u64 svc_push_pa_ceiling(void) { u64 top = READ_ONCE(gsp_sema_gr_scan_top); + u64 reg = gsp_phys_guest_ram_top(); if (top > SVC_GR_SCAN_TOP_MAX) top = SVC_GR_SCAN_TOP_MAX; if (top < SEMA_PA_MAX) top = SEMA_PA_MAX; + /* + * SESSION 153-CONTINUED-67: never let the ceiling exceed the highest + * GPA any registered guest-RAM range covers - a read above it can + * only return the backing file's padding zeros, never content. With + * the 8/10 GB guest layout (0-2 GB + 4-10 GB) the registered top is + * what it is and inside-range answers (like the r1127 stale walk at + * 0x23921d000) stay readable-but-zero; the repeated-zero + * invalidation above handles those, this bounds the rest. + */ + if (reg && reg < top) + top = reg; return top; } @@ -19947,6 +20040,62 @@ static int svc_push_page_heal(struct nv_vgpu_device *vgpu, u32 h_object, return -ENOENT; } +/* + * SESSION 153-CONTINUED-67: repeated all-zero read invalidation. + * + * The beef (direct GL 3D client) channel's pushes are re-read every pump + * pass; on the r1127/w2 boots the setup push and every draw window kept + * answering ALL-ZERO from the same cache-derived translation for the whole + * boot: "sema-pushdec: h=0xbeef0100 va=0x4001004 len=60216 src=4 + * pa=0x23921d000 ... unreadable=1 first8=00000000 00000000" and 64 + * "zero-head window ... (page mid-write; not captured)" rows - while the + * SAME VAs answered real content on other boots (r1126). A single zero + * read can be a legitimately-zero page (padding, a zeroed push slot), but + * the SAME page answering all-zero TWICE in a row through a cache-derived + * source is the stale-translation signature: the client zeroed, + * reallocated or rewrote the page and the cached pin/learned/walk answer + * keeps resolving to the dead page - every walk success re-stores its + * answer into the learned table, so nothing ever re-resolves. When the + * streak reaches two, drop the learned-cache entry (and a validated pin + * when that was the source) so the NEXT read re-runs the source chain + * (discovered map, registered-VAS walk, discovered walk, targeted + * resolve) instead of reusing the same answer; this read still returns + * what it read. Beef channels only (the GL client's push family this + * work targets); the drop is bounded and the witness bounded per boot. + */ +#define SVC_ZRW_MAX 256 +struct svc_zrw_rec { + u64 page_va; + u8 streak; +}; +static struct svc_zrw_rec svc_zrw_tab[SVC_ZRW_MAX]; +static u32 svc_zrw_next; +static u32 svc_zrw_log; + +static struct svc_zrw_rec *svc_zrw_slot(u64 page_va) +{ + u32 i; + + for (i = 0; i < SVC_ZRW_MAX; i++) + if (svc_zrw_tab[i].page_va == page_va) + return &svc_zrw_tab[i]; + i = svc_zrw_next++ % SVC_ZRW_MAX; + svc_zrw_tab[i].page_va = page_va; + svc_zrw_tab[i].streak = 0; + return &svc_zrw_tab[i]; +} + +static void svc_zrw_reset(u64 page_va) +{ + u32 i; + + for (i = 0; i < SVC_ZRW_MAX; i++) + if (svc_zrw_tab[i].page_va == page_va) { + svc_zrw_tab[i].streak = 0; + return; + } +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) @@ -20205,6 +20354,46 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, !svc_promo_lookup(page_va, &promo_pa, &promo_ap); } + /* + * SESSION 153-CONTINUED-67: repeated all-zero read + * invalidation (see the svc_zrw block before this + * function). Count consecutive all-zero answers per + * page from cache-derived sources (validated pin / + * window / trusted chain); on the second in a row + * drop the learned-cache answer (and a validated pin + * when that answered) so the next read re-walks + * instead of reusing the same dead translation. The + * current read still returns what it read. + */ + if ((ch->h_object & 0xffff0000u) == 0xbeef0000u && + src >= 2) { + if (!memchr_inv(p, 0, chunk)) { + struct svc_zrw_rec *z = + svc_zrw_slot(page_va); + + if (z->streak < 0xff) + z->streak++; + if (z->streak >= 2) { + z->streak = 0; + svc_lat_drop(page_va); + if (src == 2) + svc_pin_drop(page_va); + if (svc_zrw_log < 64) { + svc_zrw_log++; + pr_info("sema-zrw: h=0x%x va=0x%llx page=0x%llx src=%u pa=0x%llx sub=%u all-zero x2 -> drop lat%s, re-walk next read\n", + ch->h_object, + (unsigned long long)va, + (unsigned long long)page_va, + src, + (unsigned long long)pa, + svc_trust_src, + src == 2 ? "+pin" : ""); + } + } + } else { + svc_zrw_reset(page_va); + } + } done = true; } if (!done) { @@ -36445,9 +36634,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (gsp_sema_dbg && svc_pushdec_log < 2048) { svc_pushdec_log++; - pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n", + pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u sub=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n", ch->h_object, va, len, (unsigned)svc_read_last_src, + svc_trust_src, svc_read_last_pa, svc_method_stream_plausible(pb, len) ? 1 : 0, injected ? 1 : 0, -- cgit v1.2.3