vgsp-gr: m152 scratch-PT alias cache + extended pool; m153 outcome-verifying launch retry

m152: the Tier-2 shadow alias pool consumed one never-reused 128 B fetch
line per mapping per launch (512 lines/boot), so a normal launch mix
exhausted it permanently: every later launch that needed a new mapping
was refused (the deterministic r152 'launch refused: no Tier-1 mapping'
loop). Two fixes: (a) an HPA-keyed alias cache in map_shadow_locked that
returns the existing never-rewritten VA for a page already mapped, so
repeat launches of the same buffers/sem pages consume nothing; a hit
reports line_out = ~0 so contiguity bookkeeping never extends an old
line; (b) a 48-window extension pool at VA 0x04000000 (PD0 indices
32..79, one 48-page PT chunk) raising capacity to 2048 lines/boot.

m153: the intermittent per-boot mis-execution classes (engine wrote
nothing; or wrote into a lower operand window while the output operand
stayed untouched - both complete with qmd_rel/ch_rel satisfied, live
evidence in r152 smiley: passes wrote b into the a window / nothing /
correct-after-10s-retry) are now detected by the m151 retirement wait
and retried: refill the bounce window from the guest inputs, re-publish
the same method block on a fresh ring entry, re-issue schedule, re-poll,
re-wait - bounded by the timeout path's attempt budget. Gate param
gsp_phys_rmcall_gr_nowrite_retry (default 1).

3 files changed, 403 insertions, 94 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index bd70b7d..9f4bd53 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -1118,6 +1118,33 @@ bool gsp_phys_boot_requested(void)
 /* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */
 #define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE				\
 	((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
+/*
+ * m152: extended scratch-PT pool. The original 16 x 2 MB PD0 windows
+ * (512 never-reused 128 B fetch lines per boot) exhausted under a normal
+ * launch mix (each distinct guest page consumed a line per launch) and the
+ * exhaustion was permanent for the boot: every later launch that needed a
+ * new mapping was refused ("launch refused ... no Tier-1 mapping and no
+ * captured inline data"), which is the deterministic once-broken-forever
+ * failure the longevity runs kept hitting. Two changes fix the class:
+ *   (a) GR alias cache (below): a consumed fetch line is keyed by the HPA
+ *       it maps, so a repeat launch of the same buffers (or the same chunk
+ *       page / semaphore page) reuses the identical never-rewritten line and
+ *       consumes nothing;
+ *   (b) a 48-window extension pool right above the ladder's free gap
+ *       (window VA 0x04000000.., PD0 indices 32..79) backed by one 48-page
+ *       PT chunk, raising capacity to 64 windows x 32 lines = 2048.
+ * lines/boot. Both are pure capacity mechanics: no PTE and no mapping any
+ * engine may have fetched is ever rewritten.
+ */
+#define GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE	0x04000000ULL
+#define GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES	48u
+#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES				\
+	(GSP_PHYS_RMCALL_GUEST_PT_PAGES + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES)
+#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES				\
+	(GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES *				\
+	 GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
+/* HPA -> shadow-VA reuse cache; hit = no new fetch line is consumed. */
+#define GSP_PHYS_RMCALL_GR_ALIAS_CACHE		768u
 
 /*
  * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources,
@@ -1410,6 +1437,9 @@ enum gsp_phys_rmcall_buf {
 	GSP_PHYS_RMCALL_BUF_GUEST_PT13,
 	GSP_PHYS_RMCALL_BUF_GUEST_PT14,
 	GSP_PHYS_RMCALL_BUF_GUEST_PT15,
+	/* m152: extension pool PT pages (GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES
+	 * pages; allocated multi-page in alloc_chunks). */
+	GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT,
 	/*
 	 * Golden9g: the three client page-table levels the real golden VAS
 	 * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES
@@ -2181,7 +2211,18 @@ static struct gsp_phys_boot_stage {
 	bool rmcall_gr_exec_ok;		/* a real GR submission proved out */
 	u32 rmcall_gr_submit_epoch;	/* per-submission mapper epoch */
 	u32 rmcall_gr_pt_used;		/* Tier-1 window pool slots used */
-	u32 rmcall_guest_pt_win_mask;	/* Tier-2 scratch windows installed */
+	u64 rmcall_guest_pt_win_mask;	/* Tier-2 scratch windows installed */
+	/*
+	 * m152: shadow alias reuse cache: hpa -> never-rewritten shadow VA.
+	 * Populated by gsp_phys_rmcall_gr_map_shadow_locked; a hit returns the
+	 * existing line so repeat launches of the same physical pages consume
+	 * no new fetch line. Fill-order/LRU-free by design: entries are dead
+	 * weight once a client exits, but a boot has 2048 lines to give and
+	 * the cache only ever returns identical PTE content.
+	 */
+	u64 rmcall_gr_alias_hpa[GSP_PHYS_RMCALL_GR_ALIAS_CACHE];
+	u64 rmcall_gr_alias_va[GSP_PHYS_RMCALL_GR_ALIAS_CACHE];
+	u32 rmcall_gr_alias_count;
 	struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES];
 	struct gsp_phys_boot_chunk rmcall_gr_shadow;	/* Tier-2 pages */
 	/*
@@ -2894,6 +2935,7 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_gr_submit_epoch = 0;
 	boot_stage.rmcall_gr_pt_used = 0;
 	boot_stage.rmcall_guest_pt_win_mask = 0;
+	boot_stage.rmcall_gr_alias_count = 0;
 	memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt));
 	memset(&boot_stage.rmcall_gr_shadow, 0,
 	       sizeof(boot_stage.rmcall_gr_shadow));
@@ -7296,6 +7338,63 @@ static void gsp_phys_rmcall_ctx_diag_dump(const char *what,
 static void gsp_phys_rmcall_ctx_diag_decode(const char *tag);
 
 /*
+ * m152: scratch-PT pool geometry helpers. Pool local page P (0-based across
+ * the original 16-window region then the 48-window extension) maps to a PD0
+ * window VA and to a page of either the per-window single-page PT chunks
+ * (P < 16) or the multi-page extension chunk (P >= 16). Keep every
+ * consumer on these two helpers so the two regions can never drift.
+ */
+static u64 gsp_phys_rmcall_guest_pt_win_va(u32 page)
+{
+	if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES)
+		return boot_stage.rmcall_va_guest_pt + ((u64)page << 21);
+	return GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE +
+	       ((u64)(page - GSP_PHYS_RMCALL_GUEST_PT_PAGES) << 21);
+}
+
+static u8 *gsp_phys_rmcall_guest_pt_page_va(u32 page)
+{
+	if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES)
+		return gsp_phys_chunk_page_va(
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 +
+						  page], 0);
+	if (page < GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES)
+		return gsp_phys_chunk_page_va(
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT],
+			page - GSP_PHYS_RMCALL_GUEST_PT_PAGES);
+	return NULL;
+}
+
+static u64 gsp_phys_rmcall_guest_pt_page_dma(u32 page)
+{
+	if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES)
+		return (u64)gsp_phys_chunk_page_dma(
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 +
+						  page], 0);
+	return (u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT],
+		page - GSP_PHYS_RMCALL_GUEST_PT_PAGES);
+}
+
+/*
+ * PD0 index -> pool local page, or 0xffffffffu when the index is not a
+ * scratch-pool window. Consults both pool regions.
+ */
+static u32 gsp_phys_rmcall_guest_pt_page_of_pde(u32 pde_index)
+{
+	u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21);
+
+	if (pde_index >= base &&
+	    pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES)
+		return pde_index - base;
+	base = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21);
+	if (pde_index >= base &&
+	    pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES)
+		return pde_index - base + GSP_PHYS_RMCALL_GUEST_PT_PAGES;
+	return 0xffffffffu;
+}
+
+/*
  * Shared-management VAS: install the ladder's page-table window through the
  * server's own MMU walker. NV0080_CTRL_CMD_DMA_UPDATE_PDE_2 (0x80180f,
  * gpu_vaspace.c gvaspaceUpdatePde2_IMPL) takes the system-physical base of a
@@ -7849,6 +7948,9 @@ static int gsp_phys_rmcall_alloc_chunks(void)
 
 	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
 		struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[i];
+		u32 npages =
+			(i == GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT) ?
+				GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES : 1u;
 
 		if (c->pages)
 			continue;
@@ -7857,7 +7959,7 @@ static int gsp_phys_rmcall_alloc_chunks(void)
 			       i);
 			return -EINVAL;
 		}
-		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1u)) {
+		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, npages)) {
 			pr_err("vgsp-boot: rm-call: buffer %u allocation failed\n",
 			       i);
 			return -ENOMEM;
@@ -9619,6 +9721,29 @@ static void gsp_phys_rmcall_pdb_build(void)
 				   (size_t)(1 + i) * 16 + 8);
 	}
 	/*
+	 * m152: extension scratch pool - PD0 dual entries 32..79 back the
+	 * GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES pages of the multi-page extension
+	 * PT chunk at GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE (0x04000000; free
+	 * space above the GR-context windows, which end well below 0x03000000).
+	 * Same shape and same once-before-any-engine-access rule as the 16-window
+	 * loop above.
+	 */
+	for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES; i++) {
+		u64 guest_pt_pde = gsp_phys_rmcall_pde(
+			(u64)gsp_phys_chunk_page_dma(
+				&boot_stage.rmcall_chunks[
+					GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT], i));
+		u32 pd0_idx = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21) +
+			      i;
+
+		put_unaligned_le64(0,
+				   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+				   (size_t)pd0_idx * 16);
+		put_unaligned_le64(guest_pt_pde,
+				   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+				   (size_t)pd0_idx * 16 + 8);
+	}
+	/*
 	 * H1 GR context window: when the GR ladder prepared the promoted MAIN
 	 * context buffer (gsp_phys_rmcall_gr_ctx_prepare ran before this
 	 * builder), append its page tables as fresh PD0 dual entries 17.. (the
@@ -9785,11 +9910,12 @@ static void gsp_phys_rmcall_pdb_build(void)
 		src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0),
 		dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0),
 		sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0));
-	pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n",
+	pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u (+%u ext) vaBase=0x%llx extBase=0x%llx capacity=%u\n",
 		GSP_PHYS_RMCALL_GUEST_PT_PAGES,
+		GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES,
 		GSP_PHYS_RMCALL_VA_GUEST_PT_BASE,
-		GSP_PHYS_RMCALL_GUEST_PT_PAGES *
-			GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+		GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE,
+		GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES);
 }
 
 /*
@@ -15249,7 +15375,6 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa,
 					    u64 sem_hpa, u64 *src_va,
 					    u64 *dst_va, u64 *sem_va)
 {
-	struct gsp_phys_boot_chunk *guest_pt_chunk;
 	u32 line = boot_stage.rmcall_guest_pt_next_line;
 	u32 page, line_in_page;
 	u64 va;
@@ -15257,19 +15382,14 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa,
 
 	if (!boot_stage.rmcall_va_guest_pt)
 		return -EIO;
-	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
-		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
+	if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES)
 		return -ENOSPC;
 	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
 	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
-	guest_pt_chunk = &boot_stage.rmcall_chunks[
-		GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
-	if (!guest_pt_chunk->pages)
-		return -EIO;
 	/* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */
-	va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
+	va = gsp_phys_rmcall_guest_pt_win_va(page) +
 	     ((u64)line_in_page << 16);
-	pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0);
+	pt = gsp_phys_rmcall_guest_pt_page_va(page);
 	if (!pt)
 		return -EIO;
 	/*
@@ -15490,15 +15610,13 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 				if ((gsp_phys_relay_pool_exhausted & 63) == 1)
 					pr_err("vgsp-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n",
 					       gsp_phys_relay_pool_exhausted,
-					       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
-					       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+					       GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES);
 			} else {
 				gsp_phys_guest_pool_exhausted++;
 				if ((gsp_phys_guest_pool_exhausted & 63) == 1)
 					pr_err("vgsp-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n",
 					       gsp_phys_guest_pool_exhausted,
-					       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
-					       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+					       GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES);
 			}
 		} else {
 			if (relay)
@@ -16165,26 +16283,26 @@ static void __iomem *gsp_phys_gr_scratch_bar0;
 static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page)
 {
 	struct gsp_phys_rmcall_update_pde2_params p;
-	struct gsp_phys_boot_chunk *pt;
 	u32 rpc_result = 0, status = 0, pde_index;
+	u64 pt_dma;
 	int ret;
 
-	if (page >= GSP_PHYS_RMCALL_GUEST_PT_PAGES)
+	if (page >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES)
 		return -EINVAL;
 	if (!gsp_phys_rmcall_vas_shared_value())
 		return 0;
-	if (boot_stage.rmcall_guest_pt_win_mask & BIT(page))
+	if (boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page))
 		return 0;
-	pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
-	if (!pt->pages)
+	pt_dma = gsp_phys_rmcall_guest_pt_page_dma(page);
+	if (!pt_dma)
 		return -EIO;
 	if (!boot_stage.rmcall_va_guest_pt)
 		return -EIO;
-	pde_index = (u32)((boot_stage.rmcall_va_guest_pt >> 21) + page);
+	pde_index = (u32)(gsp_phys_rmcall_guest_pt_win_va(page) >> 21);
 	memset(&p, 0, sizeof(p));
 	p.pde_index = cpu_to_le32(pde_index);
 	p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
-	p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0));
+	p.pt[0].phys_addr = cpu_to_le64(pt_dma);
 	p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
 	p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
 	p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
@@ -16201,10 +16319,9 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page)
 		       ret);
 		return ret;
 	}
-	boot_stage.rmcall_guest_pt_win_mask |= BIT(page);
+	boot_stage.rmcall_guest_pt_win_mask |= BIT_ULL(page);
 	pr_info("vgsp-gr: scratch window installed pdeIndex=%u (VA 0x%llx) ptPa=0x%llx status=0x%x (Tier-2 shadow legs now reachable through the server walker)\n",
-		pde_index, (u64)pde_index << 21,
-		(u64)gsp_phys_chunk_page_dma(pt, 0), status);
+		pde_index, (u64)pde_index << 21, pt_dma, status);
 	return 0;
 }
 
@@ -16217,13 +16334,11 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page)
 static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va)
 {
 	u32 pde_index = (u32)(va >> 21);
-	u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21);
+	u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index);
 	u32 i;
 
-	if (pde_index >= base &&
-	    pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES)
-		return !!(boot_stage.rmcall_guest_pt_win_mask &
-			  BIT(pde_index - base));
+	if (page != 0xffffffffu)
+		return !!(boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page));
 	for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++)
 		if (boot_stage.rmcall_gr_win[i].used &&
 		    boot_stage.rmcall_gr_win[i].pde_index == pde_index)
@@ -16242,13 +16357,11 @@ static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va)
 static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va)
 {
 	u32 pde_index = (u32)(va >> 21);
-	u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21);
+	u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index);
 	u32 slot;
 
-	if (pde_index >= base &&
-	    pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES)
-		return gsp_phys_rmcall_gr_scratch_win_install_locked(
-			pde_index - base);
+	if (page != 0xffffffffu)
+		return gsp_phys_rmcall_gr_scratch_win_install_locked(page);
 	return gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot);
 }
 
@@ -16259,16 +16372,35 @@ static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va)
  * supplies the backing page. Called with gsp_phys_rm_lock held.
  */
 static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa,
-						u64 *va_out)
+						u64 *va_out, u32 *line_out)
 {
 	struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow;
-	struct gsp_phys_boot_chunk *pt;
 	u32 line = boot_stage.rmcall_guest_pt_next_line;
-	u32 page, line_in_page;
+	u32 page, line_in_page, i;
 	u64 va;
-	u8 *pt_va;
+	u8 *pt;
 	int ret;
 
+	/*
+	 * m152: reuse before consume. A shadow VA is a never-rewritten PTE:
+	 * once hpa X maps at VA V, every later request for X can be served by
+	 * V with byte-identical engine-visible content, so the pool only
+	 * needs one line per distinct HPA per boot instead of one per launch.
+	 * This is safe against the GMMU's no-invalidate fetch-line cache by
+	 * construction: the PTE value never differs from what the engine may
+	 * have already fetched. A hit reports line_out = ~0 so a caller that
+	 * records contiguity bookkeeping never contig-extends an old line
+	 * (a new PTE in a line the engine may have already fetched would be
+	 * invisible).
+	 */
+	for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) {
+		if (boot_stage.rmcall_gr_alias_hpa[i] == hpa) {
+			*va_out = boot_stage.rmcall_gr_alias_va[i];
+			if (line_out)
+				*line_out = 0xffffffffu;
+			return 0;
+		}
+	}
 	if (!sh->pages) {
 		ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
 					   GSP_PHYS_RMCALL_GR_SHADOW_PAGES);
@@ -16280,28 +16412,34 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa,
 		return -ENOSPC;
 	if (!boot_stage.rmcall_va_guest_pt)
 		return -EIO;
-	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
-		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
+	if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES)
 		return -ENOSPC;
 	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
 	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
 	ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page);
 	if (ret)
 		return ret;
-	pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
-	if (!pt->pages)
-		return -EIO;
-	pt_va = gsp_phys_chunk_page_va(pt, 0);
-	if (!pt_va)
+	pt = gsp_phys_rmcall_guest_pt_page_va(page);
+	if (!pt)
 		return -EIO;
 	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
-			   pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
 					    line_in_page) * 8);
 	dma_wmb();
 	boot_stage.rmcall_guest_pt_next_line = line + 1;
-	va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
+	va = gsp_phys_rmcall_guest_pt_win_va(page) +
 	     ((u64)line_in_page << 16);
 	*va_out = va;
+	if (line_out)
+		*line_out = line;
+	if (boot_stage.rmcall_gr_alias_count <
+	    GSP_PHYS_RMCALL_GR_ALIAS_CACHE) {
+		u32 ci = boot_stage.rmcall_gr_alias_count;
+
+		boot_stage.rmcall_gr_alias_hpa[ci] = hpa;
+		boot_stage.rmcall_gr_alias_va[ci] = va;
+		boot_stage.rmcall_gr_alias_count = ci + 1u;
+	}
 	return 0;
 }
 
@@ -16315,32 +16453,27 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa,
 static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte,
 						       u64 hpa, u64 *va_out)
 {
-	struct gsp_phys_boot_chunk *pt;
 	u32 page, line_in_page;
-	u8 *pt_va;
+	u8 *pt;
 	int ret;
 
 	if (pte >= GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
 		return -EINVAL;
-	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
-		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
+	if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES)
 		return -ENOSPC;
 	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
 	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
 	ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page);
 	if (ret)
 		return ret;
-	pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
-	if (!pt->pages)
-		return -EIO;
-	pt_va = gsp_phys_chunk_page_va(pt, 0);
-	if (!pt_va)
+	pt = gsp_phys_rmcall_guest_pt_page_va(page);
+	if (!pt)
 		return -EIO;
 	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
-			   pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
 					    line_in_page + pte) * 8);
 	dma_wmb();
-	*va_out = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
+	*va_out = gsp_phys_rmcall_guest_pt_win_va(page) +
 		  ((u64)line_in_page << 16) + (u64)pte * GSP_PHYS_PAGE_SIZE;
 	return 0;
 }
@@ -16621,7 +16754,7 @@ static inline u64 gsp_phys_gr_fnv1a64(const u8 *p, size_t len)
  * noinline so this rarely-taken wait does not grow the submit frame, which
  * is already near the warning bound.
  */
-static noinline void
+static noinline u32
 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req,
 			       const u32 *csum_in)
 {
@@ -16632,7 +16765,7 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req,
 	unsigned long wait_start = jiffies;
 
 	if (!req->n_data)
-		return;
+		return 0;
 	for (di = 0; di < req->n_data; di++) {
 		u32 bi = (u32)((req->data[di].reloc_va -
 				GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12);
@@ -16650,7 +16783,7 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req,
 			checked[n_all++] = bi;
 	}
 	if (!n_all)
-		return;
+		return 0;
 	for (iter = 0; iter < 300u; iter++) {
 		u32 j;
 
@@ -16684,6 +16817,41 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req,
 	}
 	pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n",
 		changed, n_all, jiffies_to_msecs(jiffies - wait_start), why);
+	return changed;
+}
+
+/*
+ * m153: fill the bounce window from the request's guest pages and capture a
+ * per-page checksum baseline. Used before the first publish and again before
+ * every outcome retry (a mis-executing first attempt can leave stale or
+ * wrong-window content behind, so a retry must start from the guest inputs
+ * again).
+ */
+static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
+				    u32 *csum_in)
+{
+	u32 di;
+
+	for (di = 0; di < req->n_data; di++) {
+		u32 bi = (u32)((req->data[di].reloc_va -
+				GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12);
+		const u8 *src;
+		u8 *dst;
+		u32 ci, csum = 0;
+
+		if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT ||
+		    !req->data[di].hpa)
+			continue;
+		src = (const u8 *)__va(req->data[di].hpa);
+		dst = gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_bounce, bi);
+		if (!dst)
+			continue;
+		memcpy(dst, src, GSP_PHYS_PAGE_SIZE);
+		for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
+			csum += get_unaligned_le32(dst + ci);
+		csum_in[bi] = csum;
+	}
+	dma_wmb();
 }
 
 int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
@@ -16833,7 +17001,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 
 			ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow,
 								   req->rel_hpa,
-								   &sh_va);
+								   &sh_va, NULL);
 			if (!ret) {
 				n_shadow++;
 				qmd_rel_hpa = req->rel_hpa;
@@ -16924,6 +17092,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 			for (k = 0; k < n_sh_rec; k++) {
 				u64 va = 0;
 
+				if (sh_rec[k].line == 0xffffffffu)
+					continue;	/* cached line: never extended */
 				if (pg_va != sh_rec[k].guest_va +
 					    (u64)sh_rec[k].npte *
 						    GSP_PHYS_PAGE_SIZE)
@@ -16950,11 +17120,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 		 */
 		if (!mapped && pg->hpa &&
 		    pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) {
-			u32 sh_line = boot_stage.rmcall_guest_pt_next_line;
+			u32 sh_line = 0xffffffffu;
 
 			if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES &&
 			    !gsp_phys_rmcall_gr_map_shadow_locked(
-				    n_shadow, pg->hpa, &sh_va)) {
+				    n_shadow, pg->hpa, &sh_va, &sh_line)) {
 				u32 k, slot = n_shadow;
 
 				mapped = true;
@@ -17241,7 +17411,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 						n_shadow,
 						(u64)gsp_phys_chunk_page_dma(sh,
 									      n_shadow),
-						&sh_va);
+						&sh_va, NULL);
 				}
 				if (!ret) {
 					bool keep_guest =
@@ -17402,7 +17572,7 @@ gr_page_unmapped:
 
 			if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES &&
 			    !gsp_phys_rmcall_gr_map_shadow_locked(
-				    n_shadow, req->qmd_hpa, &sh_va)) {
+				    n_shadow, req->qmd_hpa, &sh_va, NULL)) {
 				qmd_dst_mem = (u8 *)gsp_phys_chunk_page_va(
 						&boot_stage.rmcall_gr_shadow,
 						n_shadow) + qmd_off;
@@ -17445,7 +17615,8 @@ gr_page_unmapped:
 		u64 sh_va = 0;
 
 		ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow,
-							   req->sem_hpa, &sh_va);
+							   req->sem_hpa, &sh_va,
+							   NULL);
 		if (!ret) {
 			n_shadow++;
 			ch_rel_hpa = req->sem_hpa;
@@ -17540,29 +17711,19 @@ gr_page_unmapped:
 	if (req->n_data && boot_stage.rmcall_gr_bounce.pages) {
 		u32 dcopied = 0, dfailed = 0, di;
 
+		gsp_phys_gr_fill_bounce(req, bounce_csum_in);
 		for (di = 0; di < req->n_data; di++) {
 			u32 bi = (u32)((req->data[di].reloc_va -
 					GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >>
 				       12);
-			const u8 *src;
-			u8 *dst;
-			u32 ci = 0, csum = 0;
 
 			if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT ||
-			    !req->data[di].hpa) {
+			    !req->data[di].hpa)
 				dfailed++;
-				continue;
-			}
-			src = (const u8 *)__va(req->data[di].hpa);
-			dst = gsp_phys_chunk_page_va(
-					&boot_stage.rmcall_gr_bounce, bi);
-			memcpy(dst, src, GSP_PHYS_PAGE_SIZE);
-			for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
-				csum += get_unaligned_le32(dst + ci);
-			bounce_csum_in[bi] = csum;
-			dcopied++;
+			else if (gsp_phys_chunk_page_va(
+					 &boot_stage.rmcall_gr_bounce, bi))
+				dcopied++;
 		}
-		dma_wmb();
 		pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n",
 			dcopied, dfailed);
 		for (di = 0; di < req->n_data && di < 3; di++)
@@ -17712,9 +17873,10 @@ gr_page_unmapped:
 		boot_stage.rmcall_token,
 		qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note);
 	if (qmd_win_diag)
-		pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n",
+		pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n",
 			qmd_dst, qmd_win_pde, qmd_win_installed,
-			boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret);
+			(unsigned long long)boot_stage.rmcall_guest_pt_win_mask,
+			qmd_win_ret);
 
 	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS);
 	for (;;) {
@@ -18138,9 +18300,128 @@ gr_page_unmapped:
 	 * launch's copy-in would also race that write tail. Success path
 	 * only - the timeout/fault paths never copied out. See
 	 * gsp_phys_gr_output_retire_wait().
+	 *
+	 * m153: outcome-verifying retry. The intermittent per-boot
+	 * mis-execution classes (engine wrote nothing at all; or it wrote
+	 * into a lower operand window while the highest - output - operand
+	 * stayed untouched) both complete with qmd_rel/ch_rel satisfied, so
+	 * they cannot be caught by the completion poll. The retirement wait
+	 * is the only execution witness. When it saw zero writes, or writes
+	 * that skipped the highest-reloc operand (n_data >= 2), refill the
+	 * bounce window from the guest inputs, re-publish the same method
+	 * block on a fresh ring entry, re-issue the schedule and re-poll,
+	 * then re-wait - bounded by the same attempt budget as the timeout
+	 * path. After the budget the launch proceeds to copy-out with
+	 * whatever the engine produced (a genuinely read-only kernel is
+	 * indistinguishable here and simply pays the bounded extra polls).
 	 */
-	if (req->n_data && boot_stage.rmcall_gr_bounce.pages)
-		gsp_phys_gr_output_retire_wait(req, bounce_csum_in);
+	if (req->n_data && boot_stage.rmcall_gr_bounce.pages) {
+		u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in);
+
+		if (gsp_phys_rmcall_gr_nowrite_retry_value()) {
+			u64 top_reloc = 0;
+			u32 top_bi = 0xffffffffu, di2;
+
+			for (di2 = 0; di2 < req->n_data; di2++) {
+				u64 rv = req->data[di2].reloc_va;
+				u32 bi2;
+
+				if (!rv || rv <= top_reloc)
+					continue;
+				bi2 = (u32)((rv -
+					     GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >>
+					    12);
+				if (bi2 >= GSP_PHYS_RMCALL_GR_MAIL_SLOT)
+					continue;
+				top_reloc = rv;
+				top_bi = bi2;
+			}
+			while (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) {
+				bool top_changed = false;
+
+				if (top_bi != 0xffffffffu) {
+					const u8 *tp =
+						gsp_phys_chunk_page_va(
+							&boot_stage
+								 .rmcall_gr_bounce,
+							top_bi);
+
+					if (tp) {
+						u32 ti, tc = 0;
+
+						for (ti = 0;
+						     ti < GSP_PHYS_PAGE_SIZE;
+						     ti += 4)
+							tc += get_unaligned_le32(
+								tp + ti);
+						top_changed =
+							(tc !=
+							 bounce_csum_in[top_bi]);
+					}
+				}
+				if (wrote > 0 &&
+				    (req->n_data < 2 || top_changed))
+					break;
+				attempts++;
+				if (attempts > GSP_PHYS_RMCALL_GR_ATTEMPTS)
+					break;
+				pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u top_bi=%u top_changed=%u): refill + re-publish\n",
+					attempts, GSP_PHYS_RMCALL_GR_ATTEMPTS,
+					wrote, top_bi, top_changed);
+				gsp_phys_gr_fill_bounce(req, bounce_csum_in);
+				mutex_lock(&gsp_phys_rm_lock);
+				ret = gsp_phys_rmcall_publish_locked(
+					bar0, boot_stage.rmcall_va_pb,
+					words, &sub);
+				if (!ret)
+					ret = gsp_phys_rmcall_gr_resched_locked(
+						bar0);
+				mutex_unlock(&gsp_phys_rm_lock);
+				if (ret) {
+					pr_info("vgsp-gr: outcome retry %u publish/resched failed rc=%d (accepting current outcome)\n",
+						attempts, ret);
+					ret = 0;
+					break;
+				}
+				{
+					unsigned long dl = jiffies +
+						msecs_to_jiffies(
+						 GSP_PHYS_RMCALL_GR_RETRY_MS);
+					u32 rw0 = 0, rw1 = 0;
+
+					for (;;) {
+						if (qmd_rel_hpa)
+							rw0 = READ_ONCE(
+							  *(u32 *)((u8 *)__va(
+								qmd_rel_hpa) +
+								qmd_rel_off));
+						else if (qmd_rel)
+							rw0 = READ_ONCE(
+							  *(u32 *)sem_va);
+						if (ch_rel_hpa)
+							rw1 = READ_ONCE(
+							  *(u32 *)((u8 *)__va(
+								ch_rel_hpa) +
+								ch_rel_off));
+						else
+							rw1 = READ_ONCE(
+							  *(u32 *)(sem_va + 4));
+						if ((!qmd_rel ||
+						     rw0 == want_qmd) &&
+						    rw1 == want_ch)
+							break;
+						if (time_after_eq(jiffies, dl))
+							break;
+						msleep(GSP_PHYS_RMCALL_POLL_MS);
+					}
+				}
+				wrote = gsp_phys_gr_output_retire_wait(
+					req, bounce_csum_in);
+				pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n",
+					attempts, wrote);
+			}
+		}
+	}
 	/*
 	 * m114: the kernel ran against the pre-commit bounce window; copy the
 	 * results back to the guest's real pages (identity HPAs in the
@@ -19280,8 +19561,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 			   fwd_cand_none, fwd_cand_sys, fwd_cand_mixed,
 			   fwd_cand_vid, fwd_resolve_fail,
 			   boot_stage.rmcall_guest_pt_next_line,
-			   GSP_PHYS_RMCALL_GUEST_PT_PAGES *
-				   GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+			   GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES);
 	}
 	{
 		/*
@@ -20913,8 +21193,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296
 	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
 	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
-static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33,
-	      "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)");
+static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 34,
+	      "the rm-call ladder registers exactly thirty-four DMA buffers (14 ladder + 16 single-page guest scratch PTs + 1 extension scratch-PT chunk + 3 golden9g VAS levels)");
 static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
 	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
 	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index c5419a5..9b7235b 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -949,6 +949,31 @@ int gsp_phys_rmcall_gr_skedrec_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_skedrec_value);
 
 /*
+ * m153 no-write/wrong-place launch retry (gsp_phys_rmcall_gr_nowrite_retry;
+ * default 1). After a forwarded launch reports completion, the output
+ * retirement wait measures how many resolved data pages the engine actually
+ * changed. Two mis-execution signatures are known to be intermittent per
+ * boot: the engine writes nothing at all, or it writes into a lower operand
+ * window while the highest (output) operand stays untouched (a stale
+ * constant-source state). Both re-execute correctly on a fresh ring entry
+ * often enough to be worth a bounded retry: refill the bounce window from
+ * the guest inputs, re-publish the same method block, re-issue the schedule
+ * and re-poll, then re-wait. Set to 0 to disable the retry and accept the
+ * first outcome unconditionally.
+ */
+static int gsp_phys_rmcall_gr_nowrite_retry_param = 1;
+module_param_named(gsp_phys_rmcall_gr_nowrite_retry,
+		   gsp_phys_rmcall_gr_nowrite_retry_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_nowrite_retry,
+		 "Re-execute a forwarded GR launch whose retirement wait saw zero/wrong-place engine writes (default 1)");
+
+int gsp_phys_rmcall_gr_nowrite_retry_value(void)
+{
+	return gsp_phys_rmcall_gr_nowrite_retry_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_nowrite_retry_value);
+
+/*
  * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When
  * set, the forwarded GR launch copies the QMD's program/param pages into the
  * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index f93603c..73b0822 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -365,6 +365,10 @@ unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_value(void);
 /* m72: log SKED CWD/HWW register transitions during the launch poll. */
 int gsp_phys_rmcall_gr_skedrec_value(void);
 
+/* m153: re-execute a forwarded launch whose retirement wait saw zero or
+ * wrong-place engine writes (default 1 = enabled). */
+int gsp_phys_rmcall_gr_nowrite_retry_value(void);
+
 /* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */
 int gsp_phys_rmcall_gr_inst_sys_value(void);