vgsp: svc_lat_drop + trust sub-source tag (sub= in sema-pushdec) + SVC_ZRW zero-streak table (src>=2 all-zero x2 -> drop lat/re-walk, pin when src==2), gl_win_start arms only on hdr!=0 (root cause of 5-boot capture drought: window armed on zero no-op), svc_gl_flush zero-lead advance, svc_push_pa_ceiling from gsp_phys_guest_ram_top - capture drought BROKEN (r1131 win=1429 words, 1007 section captures, 0 zero-head refusals); MME-PRELUDE ships in one launch FIRST TIME (gen=4 words=901 + window=390 -> GR LAUNCH words=1348); stores still 0 (engine fetch NONE, MISSING_MACRO_DATA ESR 0x80000001; 8.8s fallback); next: one-variable attribution (sa_stub=0 then surf_prelude=0) + same-generation SA/program capture - SESSION 153 r1129-r1131

3 files changed, 243 insertions, 9 deletions

diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index e9c0c5a..f1d72f7 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -1954,6 +1954,41 @@ bool gsp_phys_guest_ram_active(void)
 EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active);
 
 /*
+ * Highest GPA any registered guest-RAM range covers. Conservative global
+ * bound (max across the legacy and every per-mdev record) for consumers that
+ * only need to reject translations pointing past RAM - e.g. the sema push
+ * read's PA ceiling (r1127: a stale VAS walk answered 0x23921d000 = 8.89 GB
+ * for the beef setup push; with the 8/10 GB guest layout the answer is
+ * inside the registered ranges and reads padding zeros, so no bound alone
+ * fixes it - the repeated-zero invalidation does - but any answer ABOVE
+ * this top can never read content). 0 when nothing is registered.
+ */
+u64 gsp_phys_guest_ram_top(void)
+{
+	u64 top = 0;
+	unsigned int i, j;
+
+	mutex_lock(&guest_ram_lock);
+	for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS + 1; i++) {
+		const struct gsp_phys_guest_ram_rec *rec =
+			(i == 0) ? &guest_ram_legacy : &guest_ram_mdevs[i - 1];
+
+		if (!rec->file)
+			continue;
+		for (j = 0; j < rec->ranges_n; j++) {
+			u64 end = rec->ranges[j].gpa_base +
+				  rec->ranges[j].size;
+
+			if (end > top)
+				top = end;
+		}
+	}
+	mutex_unlock(&guest_ram_lock);
+	return top;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_top);
+
+/*
  * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag,
  * path, range count and every range (gpa/offset/size). With only the legacy
  * record configured (the one-mdev campaign) the output is byte-identical to
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 3993c6e..4d9f47b 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -516,6 +516,15 @@ bool gsp_phys_guest_ram_active(void);
 void gsp_phys_guest_ram_state(char *buf, size_t len);
 
 /*
+ * Highest GPA any registered guest-RAM range covers (max of gpa_base+size
+ * across every active record; 0 when nothing is registered). A GPA at or
+ * above this can never hold guest content - the backing file's padding
+ * reads as zeros - so translation consumers use it to reject walk/pin
+ * answers that point past the registered RAM.
+ */
+u64 gsp_phys_guest_ram_top(void);
+
+/*
  * Open-mdev helpers for the per-mdev debugfs selector and the single-guest
  * guards: count the mdevs whose VFIO device is open, or resolve one open
  * mdev by its device name (the mdev UUID). The returned device is not
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 2b435c4..748eae9 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3375,6 +3375,27 @@ static void svc_lat_store(u64 va, u64 addr, int ap)
 }
 
 /*
+ * SESSION 153-CONTINUED-67: invalidate a learned-cache entry so the next
+ * lookup of this page re-runs the source chain underneath it (discovered
+ * map, registered-VAS walk, discovered walk, targeted resolve) instead of
+ * reusing a cached answer. Used by the repeated all-zero read
+ * invalidation in svc_read_va_push_ex: every walk success re-stores the
+ * answer here (svc_lat_store after each svc_lookup_trusted sub-source),
+ * so a stale walk result otherwise shadows a fresh walk for the rest of
+ * the boot (r1127: the beef setup push kept answering the same zero page
+ * while the client's content was live elsewhere).
+ */
+static void svc_lat_drop(u64 va)
+{
+	struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];
+
+	if (l->valid &&
+	    (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
+	    (va & ~(u64)(SEMA_PAGE_SIZE - 1)))
+		l->valid = false;
+}
+
+/*
  * r820: driver-declared kernel-buffer translations (PROMOTE_CTX).
  *
  * The guest RM announces the physical backing of a channel's context
@@ -4293,11 +4314,22 @@ static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 	return -ENOENT;
 }
 
+/*
+ * SESSION 153-CONTINUED-67: which sub-source of svc_lookup_trusted answered
+ * the last call (1=channel VAS 2=pin 3=learned 4=discovered map 5=VAS walk
+ * 6=discovered walk 7=targeted 8=promote 0=miss). Printed as `sub=` by the
+ * pushdec witness so a repeated-zero read can be attributed to the exact
+ * layer (the r1127 investigation could not tell a stale learned entry from
+ * a stale registered-VAS walk).
+ */
+static u8 svc_trust_src;
+
 static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 			      int *ap)
 {
 	u64 pa = 0;
 
+	svc_trust_src = 0;
 	/*
 	 * r835: the decoded channel's own page tables answer first. When a
 	 * channel is being serviced (svc_cur_vas set) its VAS is the only
@@ -4305,27 +4337,35 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 	 * hold another client's pages for the same numeric VA. A failed
 	 * walk falls through to the old chain unchanged.
 	 */
-	if (!svc_chanvas_lookup(vgpu, va, addr, ap))
+	if (!svc_chanvas_lookup(vgpu, va, addr, ap)) {
+		svc_trust_src = 1;
 		return 0;
+	}
 	if (!svc_pin_lookup_any(va, addr, ap)) {
+		svc_trust_src = 2;
 		return 0;
 	}
-	if (!svc_lat_lookup(va, addr, ap))
+	if (!svc_lat_lookup(va, addr, ap)) {
+		svc_trust_src = 3;
 		return 0;
+	}
 	if (!svc_map_lookup(va, &pa)) {
 		*addr = pa;
 		*ap = SVC_APERTURE_SYS;
 		svc_lat_store(va, pa, SVC_APERTURE_SYS);
+		svc_trust_src = 4;
 		return 0;
 	}
 	if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) {
 		*addr += va & 0xfff;
 		svc_lat_store(va, *addr, *ap);
+		svc_trust_src = 5;
 		return 0;
 	}
 	if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
 		*addr += va & 0xfff;
 		svc_lat_store(va, *addr, *ap);
+		svc_trust_src = 6;
 		return 0;
 	}
 	if (!svc_pd_found) {
@@ -4340,6 +4380,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 		if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
 			*addr += va & 0xfff;
 			svc_lat_store(va, *addr, *ap);
+			svc_trust_src = 6;
 			return 0;
 		}
 	}
@@ -4348,6 +4389,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 		*addr = pa;
 		*ap = SVC_APERTURE_SYS;
 		svc_lat_store(va, pa, SVC_APERTURE_SYS);
+		svc_trust_src = 7;
 		return 0;
 	}
 	/*
@@ -4365,6 +4407,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 	 */
 	if (!svc_promo_lookup(va, addr, ap)) {
 		svc_lat_store(va, *addr, *ap);
+		svc_trust_src = 8;
 		return 0;
 	}
 	return -ENOENT;
@@ -9390,6 +9433,21 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 				 * when the window head is a real word - a
 				 * later pass/entry on the filled page captures
 				 * the frame.
+				 *
+				 * SESSION 153-CONTINUED-67: the w2/r1127
+				 * zero-head refusals were NOT mid-write
+				 * windows that later filled: the same word at
+				 * the same offset read zero across every pass
+				 * of the boot (64 refusals, census content
+				 * stable at 0x1f8..0x614), because the window
+				 * start had been armed on a zero word (a
+				 * no-op INC(0) slot) in the gap after the last
+				 * foreign-subchannel run. A zero word is not a
+				 * method: advance past the leading zero words
+				 * to the first real word and capture the
+				 * window from there. Only a window that is
+				 * ENTIRELY zero is still refused (the r1031
+				 * protection).
 				 */
 				if (at_off > svc_gl.start_off &&
 				    svc_gl.start_off + 4u <= len) {
@@ -9397,16 +9455,39 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 
 					memcpy(&w0h, buf + svc_gl.start_off, 4);
 					if (!w0h) {
-						if (gsp_sema_dbg &&
-						    svc_gl_np_log < 64) {
+						u32 adv = svc_gl.start_off;
+						u32 lim = min_t(u32, at_off,
+								len);
+
+						while (adv + 4u <= lim) {
+							memcpy(&w0h,
+							       buf + adv, 4);
+							if (w0h)
+								break;
+							adv += 4u;
+						}
+						if (!w0h) {
+							if (gsp_sema_dbg &&
+							    svc_gl_np_log < 64) {
+								svc_gl_np_log++;
+								pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (whole window zero; not captured)\n",
+									ch->h_object,
+									push_va,
+									svc_gl.start_off,
+									(at_off - svc_gl.start_off) / 4u);
+							}
+							goto out;
+						}
+						if (svc_gl_np_log < 64) {
 							svc_gl_np_log++;
-							pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (page mid-write; not captured)\n",
+							pr_info("sema-gl: zero-lead window h=0x%x push=0x%llx start=0x%x->0x%x win=%u words (skipped zero no-ops; capturing)\n",
 								ch->h_object,
 								push_va,
 								svc_gl.start_off,
-								(at_off - svc_gl.start_off) / 4u);
+								adv,
+								(at_off - adv) / 4u);
 						}
-						goto out;
+						svc_gl.start_off = adv;
 					}
 				}
 				/*
@@ -15581,7 +15662,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			if (as != 0xffu) {
 				if (subch != as)
 					gl_win_start = 0;
-				else if (!gl_win_start &&
+				else if (!gl_win_start && hdr &&
 					 !(op == 1 && addr >= 0x1b00 &&
 					   addr <= 0x1b0c))
 					gl_win_start = off;
@@ -19844,11 +19925,23 @@ static u64 svc_lastgood_ring_pa;
 static u64 svc_push_pa_ceiling(void)
 {
 	u64 top = READ_ONCE(gsp_sema_gr_scan_top);
+	u64 reg = gsp_phys_guest_ram_top();
 
 	if (top > SVC_GR_SCAN_TOP_MAX)
 		top = SVC_GR_SCAN_TOP_MAX;
 	if (top < SEMA_PA_MAX)
 		top = SEMA_PA_MAX;
+	/*
+	 * SESSION 153-CONTINUED-67: never let the ceiling exceed the highest
+	 * GPA any registered guest-RAM range covers - a read above it can
+	 * only return the backing file's padding zeros, never content. With
+	 * the 8/10 GB guest layout (0-2 GB + 4-10 GB) the registered top is
+	 * what it is and inside-range answers (like the r1127 stale walk at
+	 * 0x23921d000) stay readable-but-zero; the repeated-zero
+	 * invalidation above handles those, this bounds the rest.
+	 */
+	if (reg && reg < top)
+		top = reg;
 	return top;
 }
 
@@ -19947,6 +20040,62 @@ static int svc_push_page_heal(struct nv_vgpu_device *vgpu, u32 h_object,
 	return -ENOENT;
 }
 
+/*
+ * SESSION 153-CONTINUED-67: repeated all-zero read invalidation.
+ *
+ * The beef (direct GL 3D client) channel's pushes are re-read every pump
+ * pass; on the r1127/w2 boots the setup push and every draw window kept
+ * answering ALL-ZERO from the same cache-derived translation for the whole
+ * boot: "sema-pushdec: h=0xbeef0100 va=0x4001004 len=60216 src=4
+ * pa=0x23921d000 ... unreadable=1 first8=00000000 00000000" and 64
+ * "zero-head window ... (page mid-write; not captured)" rows - while the
+ * SAME VAs answered real content on other boots (r1126). A single zero
+ * read can be a legitimately-zero page (padding, a zeroed push slot), but
+ * the SAME page answering all-zero TWICE in a row through a cache-derived
+ * source is the stale-translation signature: the client zeroed,
+ * reallocated or rewrote the page and the cached pin/learned/walk answer
+ * keeps resolving to the dead page - every walk success re-stores its
+ * answer into the learned table, so nothing ever re-resolves. When the
+ * streak reaches two, drop the learned-cache entry (and a validated pin
+ * when that was the source) so the NEXT read re-runs the source chain
+ * (discovered map, registered-VAS walk, discovered walk, targeted
+ * resolve) instead of reusing the same answer; this read still returns
+ * what it read. Beef channels only (the GL client's push family this
+ * work targets); the drop is bounded and the witness bounded per boot.
+ */
+#define SVC_ZRW_MAX 256
+struct svc_zrw_rec {
+	u64 page_va;
+	u8 streak;
+};
+static struct svc_zrw_rec svc_zrw_tab[SVC_ZRW_MAX];
+static u32 svc_zrw_next;
+static u32 svc_zrw_log;
+
+static struct svc_zrw_rec *svc_zrw_slot(u64 page_va)
+{
+	u32 i;
+
+	for (i = 0; i < SVC_ZRW_MAX; i++)
+		if (svc_zrw_tab[i].page_va == page_va)
+			return &svc_zrw_tab[i];
+	i = svc_zrw_next++ % SVC_ZRW_MAX;
+	svc_zrw_tab[i].page_va = page_va;
+	svc_zrw_tab[i].streak = 0;
+	return &svc_zrw_tab[i];
+}
+
+static void svc_zrw_reset(u64 page_va)
+{
+	u32 i;
+
+	for (i = 0; i < SVC_ZRW_MAX; i++)
+		if (svc_zrw_tab[i].page_va == page_va) {
+			svc_zrw_tab[i].streak = 0;
+			return;
+		}
+}
+
 static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
 			       struct svc_channel *ch, u64 va, void *buf,
 			       size_t len, bool skip_ipin)
@@ -20205,6 +20354,46 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
 					!svc_promo_lookup(page_va, &promo_pa,
 							  &promo_ap);
 			}
+			/*
+			 * SESSION 153-CONTINUED-67: repeated all-zero read
+			 * invalidation (see the svc_zrw block before this
+			 * function). Count consecutive all-zero answers per
+			 * page from cache-derived sources (validated pin /
+			 * window / trusted chain); on the second in a row
+			 * drop the learned-cache answer (and a validated pin
+			 * when that answered) so the next read re-walks
+			 * instead of reusing the same dead translation. The
+			 * current read still returns what it read.
+			 */
+			if ((ch->h_object & 0xffff0000u) == 0xbeef0000u &&
+			    src >= 2) {
+				if (!memchr_inv(p, 0, chunk)) {
+					struct svc_zrw_rec *z =
+						svc_zrw_slot(page_va);
+
+					if (z->streak < 0xff)
+						z->streak++;
+					if (z->streak >= 2) {
+						z->streak = 0;
+						svc_lat_drop(page_va);
+						if (src == 2)
+							svc_pin_drop(page_va);
+						if (svc_zrw_log < 64) {
+							svc_zrw_log++;
+							pr_info("sema-zrw: h=0x%x va=0x%llx page=0x%llx src=%u pa=0x%llx sub=%u all-zero x2 -> drop lat%s, re-walk next read\n",
+								ch->h_object,
+								(unsigned long long)va,
+								(unsigned long long)page_va,
+								src,
+								(unsigned long long)pa,
+								svc_trust_src,
+								src == 2 ? "+pin" : "");
+						}
+					}
+				} else {
+					svc_zrw_reset(page_va);
+				}
+			}
 			done = true;
 		}
 		if (!done) {
@@ -36445,9 +36634,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					 */
 					if (gsp_sema_dbg && svc_pushdec_log < 2048) {
 						svc_pushdec_log++;
-						pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n",
+						pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u sub=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n",
 							ch->h_object, va, len,
 							(unsigned)svc_read_last_src,
+							svc_trust_src,
 							svc_read_last_pa,
 							svc_method_stream_plausible(pb, len) ? 1 : 0,
 							injected ? 1 : 0,