vgsp: svc_push_page_heal (beef push all-zero read re-resolves via launch-page chain, pin/cache, src=10; sema-pushzero len cap 1024), mme-reloc-rsv (MME-family payload miss resolves via RELOC-FIXED ledger/pin/walk shadow-map), pin_gr_stream pushzero 4K-page bulk demand (48/batch); HW retire only with A+0x34a8+C dropped - draw payloads live in exactly the refused families (0x1c00-0x1c20 MME macro-code uploads / 0x3810 CALL_MME_MACRO); Graphics Exception ChID 0001 Class 0x0000c597 Offset 0 - SESSION 153 r1070-r1072

2 files changed, 198 insertions, 8 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index b2d759e..74ebbd5 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -20130,8 +20130,69 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out,
 					if (wv < 0x10000u || (wv & 3u))
 						continue;
 					if (!gsp_phys_rmcall_gr_3d_reloc(
-						    &up, &lo, req, n_shadow))
+						    &up, &lo, req, n_shadow)) {
+						/*
+						 * SESSION 153-CONTINUED-51:
+						 * the payload page missed the
+						 * launch's page set (the QMD
+						 * pointer region caps the set
+						 * far below a full window's
+						 * operand count - r1071: the
+						 * macro payload pages
+						 * 0x44fd200..0x44fde00 were
+						 * ~12 distinct pages while the
+						 * set held 7; the words stayed
+						 * RAW and the engine fetched an
+						 * untranslatable guest VA on
+						 * every full window - the
+						 * 0xbadf1201 priv-error
+						 * poison). Resolve the page
+						 * through the SAME trusted
+						 * ledger/pin/walk chain the
+						 * launch pages use (the
+						 * RELOC-FIXED resolver) and
+						 * shadow-map it exactly like a
+						 * matched page. A resolve/map
+						 * miss or a shadow VA that
+						 * cannot fit the 32-bit slot
+						 * stays verbatim (never
+						 * guessed).
+						 */
+						struct gsp_sema_gr_reloc r;
+						u64 va2 = (u64)wv;
+						u64 sh2 = 0;
+						u32 line2 = 0;
+
+						if (gsp_sema_gr_reloc_resolve(
+							    va2, &r) ||
+						    !r.hpa)
+							continue;
+						if (*n_shadow >=
+						    GSP_PHYS_RMCALL_GR_SHADOW_PAGES)
+							continue;
+						if (gsp_phys_rmcall_gr_map_shadow_locked(
+							    *n_shadow, r.hpa,
+							    &sh2, &line2))
+							continue;
+						if (line2 != 0xffffffffu)
+							(*n_shadow)++;
+						sh2 |= va2 &
+							(GSP_PHYS_PAGE_SIZE - 1);
+						up = (u32)(sh2 >> 32) & 0xffu;
+						lo = (wv & 0x3u) |
+							((u32)sh2 & 0xfffffffcu);
+						if (up != 0)
+							continue;
+						d[wi] = lo;
+						reloc++;
+						if (mme_log < 32) {
+							mme_log++;
+							pr_info("vgsp-gr: mme-reloc-rsv addr=0x%x wi=%u w=0x%x -> 0x%x\n",
+								addr, wi, wv,
+								lo);
+						}
 						continue;
+					}
 					if (up != 0)
 						continue;
 					d[wi] = lo;
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 789cf27..cf3bf6a 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -18924,6 +18924,101 @@ static u64 svc_push_pa_ceiling(void)
 	return top;
 }
 
+/*
+ * SESSION 153-CONTINUED-51: deterministic push-page resolution.
+ *
+ * The direct GL client's 5-7 KB window pushes routinely read back all
+ * zero through the trusted chain on some boots ("sema-pushzero ...
+ * rescued=0", r1033-class) while the SAME VA pages answer real content
+ * on other boots - the learned cache (lat) can shadow a page-table walk
+ * that has since moved, and the client reuses/reallocates its push VAs
+ * across frames. The launch-page chain (svc_gr_page_note) resolves
+ * through the ledger then a FRESH walk; the push read chain consults the
+ * learned cache FIRST, so a stale lat/pin entry shadows the fresh walk
+ * forever. When the trusted read answered an all-zero page, consult the
+ * same ledger/walk chain the launch pages use and, when a DIFFERENT page
+ * with non-zero content answers, adopt it AND heal the read sources
+ * (validated pin + lat) so every later read of this page is deterministic
+ * instead of a per-boot lottery.
+ *
+ * Only a page whose content differs from all-zero is adopted: a
+ * legitimately-zero page (padding) must never steal a correct
+ * translation. Returns 0 when a replacement page was adopted (pa_out /
+ * ap_out updated and `p` refilled), -ENOENT when nothing better exists.
+ */
+static int svc_push_page_heal(struct nv_vgpu_device *vgpu, u32 h_object,
+			      u64 va, u64 net_pa, int net_ap, u8 *p,
+			      size_t chunk, u64 *pa_out, int *ap_out)
+{
+	u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 off = va & (SEMA_PAGE_SIZE - 1);
+	u64 cur_page = net_pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 lpa = 0, wpa = 0;
+	u32 lap = 0;
+	int wap = SVC_APERTURE_SYS;
+	static u32 heal_log;
+
+	/* 1. The ledger: the launch-page chain's landing authority. A page a
+	 *    host write or an earlier launch resolution landed on is the
+	 *    page the client's own chain agreed about - the read chain does
+	 *    not consult it at all. */
+	if (!svc_ledger_lookup(page_va, &lpa, &lap) &&
+	    (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) {
+		u32 map_ap = svc_ap_of((int)lap);
+
+		if (svc_read_ap(vgpu, lpa + off,
+				map_ap == SVC_APERTURE_FB ?
+					SVC_APERTURE_FB : SVC_APERTURE_SYS,
+				p, chunk) == 0 &&
+		    memchr_inv(p, 0, chunk)) {
+			svc_pin_store(page_va, lpa & ~(u64)(SEMA_PAGE_SIZE - 1),
+				      (int)map_ap);
+			svc_lat_store(va, lpa + off, (int)map_ap);
+			*pa_out = lpa + off;
+			*ap_out = (int)map_ap;
+			if (heal_log < 64) {
+				heal_log++;
+				pr_info("sema-pushheal: h=0x%x va=0x%llx ledger pa=0x%llx (was 0x%llx, zero)\n",
+					h_object, (unsigned long long)va,
+					(unsigned long long)(lpa + off),
+					(unsigned long long)net_pa);
+			}
+			return 0;
+		}
+	}
+
+	/* 2. A FRESH walk, bypassing the learned cache: the guest may have
+	 *    remapped the VA after the lat entry was written. */
+	if (svc_walk_vaspaces(vgpu, page_va, &wpa, &wap) == 0 &&
+	    wpa != 0 && (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) {
+		u32 map_ap = svc_ap_of(wap);
+
+		if (svc_read_ap(vgpu, wpa + off,
+				map_ap == SVC_APERTURE_FB ?
+					SVC_APERTURE_FB : SVC_APERTURE_SYS,
+				p, chunk) == 0 &&
+		    memchr_inv(p, 0, chunk)) {
+			svc_pin_store(page_va, wpa & ~(u64)(SEMA_PAGE_SIZE - 1),
+				      (int)map_ap);
+			svc_lat_store(va, wpa + off, (int)map_ap);
+			*pa_out = wpa + off;
+			*ap_out = (int)map_ap;
+			if (heal_log < 64) {
+				heal_log++;
+				pr_info("sema-pushheal: h=0x%x va=0x%llx walk pa=0x%llx (was 0x%llx, zero)\n",
+					h_object, (unsigned long long)va,
+					(unsigned long long)(wpa + off),
+					(unsigned long long)net_pa);
+			}
+			return 0;
+		}
+	}
+	/* Nothing better: restore the zero state exactly (the caller's read
+	 * filled p with zeros and must see zeros back). */
+	memset(p, 0, chunk);
+	return -ENOENT;
+}
+
 static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
 			       struct svc_channel *ch, u64 va, void *buf,
 			       size_t len, bool skip_ipin)
@@ -19104,11 +19199,12 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
 						svc_read_last_pa = rpa;
 						svc_lat_store(page_va, ppa,
 							      pap);
-						if (pushzero_log < 32) {
+						if (pushzero_log < 1024) {
 							pushzero_log++;
-							pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=1\n",
+							pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=1\n",
 								ch->h_object,
 								(unsigned long long)va,
+								(unsigned)len,
 								src,
 								(unsigned long long)pa,
 								(unsigned long long)ppa);
@@ -19117,25 +19213,58 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
 						/* promo page unusable: put the
 						 * zero state back exactly. */
 						memset(p, 0, chunk);
-						if (!rrc && pushzero_log < 32) {
+						if (!rrc && pushzero_log < 1024) {
 							pushzero_log++;
-							pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n",
+							pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n",
 								ch->h_object,
 								(unsigned long long)va,
+								(unsigned)len,
 								src,
 								(unsigned long long)pa,
 								(unsigned long long)ppa);
 						}
 					}
-				} else if (pushzero_log < 32) {
+				} else if (pushzero_log < 1024) {
 					pushzero_log++;
-					pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n",
+					pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n",
 						ch->h_object,
-						(unsigned long long)va, src,
+						(unsigned long long)va,
+						(unsigned)len, src,
 						(unsigned long long)pa);
 				}
 			}
 			/*
+			 * SESSION 153-CONTINUED-51: when the zero-content
+			 * rescue above did not produce non-zero bytes, the
+			 * page must still be resolved deterministically -
+			 * run the SAME ledger/fresh-walk chain the launch
+			 * pages use (svc_gr_page_note) for this page. A
+			 * different page with non-zero content is adopted,
+			 * tagged src=10, and pinned so every later read of
+			 * the page (continuation chunks and future frames)
+			 * answers from the healed source instead of the
+			 * learned cache. This is the r1069 class: every
+			 * window push read zeros for a whole boot while the
+			 * launch pages resolved fine.
+			 */
+			if (src >= 2 &&
+			    (ch->h_object & 0xffff0000u) == 0xbeef0000u &&
+			    (first ? len >= 512u : chunk >= 512u) &&
+			    !memchr_inv(p, 0, chunk)) {
+				u64 hpa = pa;
+				int hap = ap;
+
+				if (!svc_push_page_heal(vgpu, ch->h_object,
+							va, pa, ap, p,
+							chunk, &hpa, &hap)) {
+					pa = hpa;
+					svc_read_last_src = 10;
+					svc_read_last_pa = hpa;
+				}
+				/* Not healed: the original (zero) source and
+				 * its label stay; nothing was adopted. */
+			}
+			/*
 			 * r820: name whether the trusted answer came from a
 			 * driver-declared PROMOTE_CTX buffer; the decode uses
 			 * it for the zero-NOP acceptance (see the flag).