vgsp-gr: output-footprint retirement wait + two-bank win0 rotation + page hashes

Two independent hardening changes for the per-boot 'all-wrong' class:

1. Output-footprint retirement wait (gsp_phys_gr_output_retire_wait): qmd_rel
   is a schedule/fetch ack, not a data-path retirement - the engine's stores
   into the bounce pages can land after it. Copy-out (and the next launch's
   copy-in) before those stores land loses them: the guest observes zeros
   for a completed launch (matmul csum=0, r146) or stale/mixed content
   (smiley b-into-a, r143/r145). The wait polls up to 300x5 ms, success path
   only, until every page of the output footprint (contiguous descending
   reloc run from the max reloc_va) differs from its copy-in checksum;
   logs 'output retirement wait: N/N pages changed after T ms'.

2. Two-bank win0 rotation: pool grows 8 -> 16 slots (VAs 0x1e0000..0x1effff,
   below the mail-injection slot 0x1f0000), bank selected by submission epoch
   (epoch & 1). A launch's copy into one bank can no longer clobber the
   pages the previous launch's engine index/data fetches may still read.

3. Evidence: full win0 param page dump rows + FNV-1a pre/post hashes of the
   win0 param page and the guest param page, so a torn/mixed read source is
   provable from the log on the next occurrence.

static_asserts: pool < mail slot and SLOTS == 2 x BANK. Build clean.
Live validation pending (next cycle: mail->matmul->smiley->victory_math->
vector_add).

1 files changed, 211 insertions, 27 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index f83899d..af7befc 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -999,24 +999,34 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_VA_DST		(GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE)
 #define GSP_PHYS_RMCALL_VA_SEM		(GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE)
 /*
- * m69: window-0 shadow pool - 8 module pages holding the Tier-2 copies of
+ * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of
  * the QMD's program/param pages, so the GR engine never has to read through
  * the window-1 scratch VA range whose PT chain (DMA_UPDATE_PDE_2
- * pdeIndex=1) has never been proven engine-readable.
+ * pdeIndex=1) has never been proven engine-readable. The pool is split
+ * into two 8-page banks; the forwarded launch alternates banks per launch
+ * (m151) so one launch's copy-in can never clobber the pages the previous
+ * launch's engine fetches may still be reading (late-fetch safety).
  *
  * The base MUST stay clear of the m114/m150 GR data bounce window
  * (GSP_PHYS_RMCALL_GR_BOUNCE_VA0 .. +PAGES, main-PT slots 0x110..0x1df):
  * this pool is re-planted on every gsp_phys_rmcall_pdb_build() while the
  * bounce PTEs are a one-shot plant, so any overlap lets the later win0
  * plant steal the bounce slots and the engine writes results into the
- * shadow pages, losing them. 0x1e0000 (slots 0x1e0..0x1e7) sits directly
- * above the last bounce/mail/marker page (0x1df000) and below the guest
- * scratch window (0x200000). Keep it that way (see the static_assert next
- * to the bounce constants).
+ * shadow pages, losing them. 0x1e0000 (slots 0x1e0..0x1ef) sits directly
+ * above the last bounce/mail/marker page (0x1df000) and below the
+ * mail-injection slot 0x1f0 and the guest scratch window (0x200000). Keep
+ * it that way (see the static_assert next to the bounce constants).
  */
 #define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \
 	(GSP_PHYS_RMCALL_VA_BASE + 0xE0000ULL)
-#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS	8u
+#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS	16u
+/*
+ * m151: one bank = one launch's worth of QMD shadow pages. A forwarded
+ * launch consumes at most eight pages (param via qmd_word 32, program 48,
+ * bank 0/1/2/3/5/7 at qmd_word 34..46 even); the per-launch bounds check
+ * uses this so a launch can never spill into the other bank.
+ */
+#define GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS	8u
 /*
  * m99: engine-tree branches for identity-mapping the forwarded kernel's
  * guest data pages. The walker uses our root[0] -> PD2 -> PD1 -> PD0 dual ->
@@ -2176,12 +2186,15 @@ static struct gsp_phys_boot_stage {
 	struct gsp_phys_boot_chunk rmcall_gr_shadow;	/* Tier-2 pages */
 	/*
 	 * m69: window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0=1).
-	 * Eight module pages whose PTEs are planted in the proven window-0
-	 * PT (slots derive from GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) by
-	 * gsp_phys_rmcall_pdb_build, before the engine's first fetch (same
-	 * pattern as the qmdwin slot). The launch copies the QMD's
-	 * program/param page content into them and rewrites the QMD pointers
-	 * there, so the GR engine never reads the window-1 scratch range.
+	 * Sixteen module pages (two 8-page banks; m151 alternates the bank
+	 * per launch so a launch's copy-in cannot clobber pages the previous
+	 * launch's engine fetches may still be reading) whose PTEs are
+	 * planted in the proven window-0 PT (slots derive from
+	 * GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) by gsp_phys_rmcall_pdb_build,
+	 * before the engine's first fetch (same pattern as the qmdwin slot).
+	 * The launch copies the QMD's program/param page content into them
+	 * and rewrites the QMD pointers there, so the GR engine never reads
+	 * the window-1 scratch range.
 	 */
 	struct gsp_phys_boot_chunk rmcall_gr_shadow_win0;
 	u32 rmcall_gr_shadow_win0_used;
@@ -2231,6 +2244,17 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
 	      GSP_PHYS_RMCALL_GR_BOUNCE_VA0 +
 	      (u64)GSP_PHYS_RMCALL_GR_BOUNCE_PAGES * GSP_PHYS_PAGE_SIZE,
 	      "win0 shadow pool must not overlap the GR data bounce window");
+/*
+ * m151: and the two-bank pool must end below the reserved mail-injection
+ * slot 0x1f0 (VA 0x1f0000) so the extra slots never steal that PTE.
+ */
+static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+	      (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <=
+	      GSP_PHYS_RMCALL_VA_BASE + 0xF0000ULL,
+	      "win0 shadow pool must end below the reserved mail-injection slot");
+static_assert(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS ==
+	      2u * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS,
+	      "win0 shadow pool must be exactly two banks");
 	/* m99: guest data identity-map table branches (see the PD*_MAX block). */
 	struct gsp_phys_boot_chunk
 		rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX];
@@ -9467,12 +9491,18 @@ static void gsp_phys_rmcall_pdb_build(void)
 					(size_t)idx * 8);
 			}
 			dma_wmb();
-			pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x1e0..0x1e7, before first fetch)\n",
+			pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x%x..0x%x, before first fetch)\n",
 				GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS,
 				GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE,
 				GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
 					(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) *
-						GSP_PHYS_PAGE_SIZE);
+						GSP_PHYS_PAGE_SIZE,
+				(u32)((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >> 12) &
+				      0x1ffu),
+				(u32)(((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+					(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) *
+						GSP_PHYS_PAGE_SIZE) >> 12) &
+				      0x1ffu));
 		}
 	}
 	/*
@@ -16553,6 +16583,103 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0)
 	return 0;
 }
 
+/*
+ * m151: FNV-1a 64-bit over one page. Used to compare the win0 param page
+ * and the guest param page before and after the poll, so a launch that
+ * wrote (or clobbered) the page the engine read is provable from the log.
+ */
+static inline u64 gsp_phys_gr_fnv1a64(const u8 *p, size_t len)
+{
+	u64 h = 0xcbf29ce484222325ULL;
+	size_t i;
+
+	for (i = 0; i < len; i++) {
+		h ^= p[i];
+		h *= 0x100000001b3ULL;
+	}
+	return h;
+}
+
+/*
+ * m151: output-footprint retirement wait. qmd_rel is a schedule/fetch ack
+ * (the front end latched and fetched the QMD), not a data-path retirement:
+ * the SM's stores into the bounce pages can still be in flight when the ack
+ * lands. Copying out - and the next launch's copy-in - before those stores
+ * land loses them: the guest observes zeros for a completed launch, or
+ * stale/mixed content when the next copy-in races the previous engine's
+ * write tail. Wait (bounded at 300 x 5 ms) until every page of the output
+ * footprint has changed from its copy-in checksum. noinline so this
+ * rarely-taken wait does not grow the submit frame, which is already near
+ * the warning bound.
+ */
+static noinline void
+gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req,
+			       const u32 *csum_in)
+{
+	u64 out_base = 0;
+	u32 out_pages = 0, di;
+
+	for (di = 0; di < req->n_data; di++)
+		if (req->data[di].reloc_va > out_base)
+			out_base = req->data[di].reloc_va;
+	if (out_base) {
+		for (;;) {
+			u64 want = out_base -
+				(u64)out_pages * GSP_PHYS_PAGE_SIZE;
+			u32 k;
+			bool hit = false;
+
+			for (k = 0; k < req->n_data; k++)
+				if (req->data[k].reloc_va == want) {
+					hit = true;
+					break;
+				}
+			if (!hit ||
+			    out_pages >= GSP_PHYS_RMCALL_GR_MAIL_SLOT)
+				break;
+			out_pages++;
+		}
+	}
+	if (!out_pages)
+		out_pages = 1u;
+	else if (out_pages > GSP_PHYS_RMCALL_GR_MAIL_SLOT)
+		out_pages = GSP_PHYS_RMCALL_GR_MAIL_SLOT;
+	if (out_pages >= 2u) {
+		unsigned long wait_start = jiffies;
+		u32 changed = 0, iter;
+
+		for (iter = 0; iter < 300u; iter++) {
+			u32 j;
+
+			changed = 0;
+			for (j = 0; j < out_pages; j++) {
+				u64 va = out_base -
+					(u64)j * GSP_PHYS_PAGE_SIZE;
+				u32 bi = (u32)((va -
+					GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12);
+				const u8 *src;
+				u32 ci, csum = 0;
+
+				if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT)
+					continue;
+				src = gsp_phys_chunk_page_va(
+					&boot_stage.rmcall_gr_bounce, bi);
+				for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4)
+					csum += get_unaligned_le32(src + ci);
+				if (csum != csum_in[bi])
+					changed++;
+			}
+			if (changed >= out_pages)
+				break;
+			usleep_range(4000, 6000);
+		}
+		pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n",
+			changed, out_pages,
+			jiffies_to_msecs(jiffies - wait_start),
+			(changed >= out_pages) ? "" : " (TIMEOUT)");
+	}
+}
+
 int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				     struct gsp_phys_backend_copy_result *out)
 {
@@ -16575,6 +16702,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	unsigned long deadline;
 	u8 *sem_va;
 	u32 epoch, words, i, n_shadow = 0;
+	u32 win0_bank_base = 0, shadow_slot_limit = 0;
+	const u8 *guest_param_pg;
 	u32 want_qmd = 0, want_ch = 0;
 	bool qmd_rel = false, ch_rel = false;
 	u64 qmd_rel_hpa = 0, ch_rel_hpa = 0;
@@ -16598,6 +16727,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_MAIL_SLOT];
 	sked_tick = 0;
 	sked_first = true;
+	guest_param_pg = NULL;
 	u32 qmd_win_pde = 0, qmd_win_installed = 0;
 	int qmd_win_ret = 0;
 	bool qmd_win_diag = false;
@@ -16647,8 +16777,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	 * the program page (silent corruption; no launch can survive it).
 	 * The win0 PTEs are planted once at stage 3; reusing the same VAs
 	 * with fresh content is the proven pattern.
+	 *
+	 * m151: the cursor now selects one of two 8-page banks, alternating
+	 * with the submission epoch (set below). The engine's index/data
+	 * fetches of the previous launch's QMD shadow pages can still be in
+	 * flight when this launch runs, so the bank alternation protects
+	 * them from this launch's copy-in (late-fetch safety).
 	 */
-	boot_stage.rmcall_gr_shadow_win0_used = 0;
 	mutex_lock(&gsp_phys_rm_lock);
 	if (!gsp_phys_backend_gr_ready()) {
 		mutex_unlock(&gsp_phys_rm_lock);
@@ -16659,6 +16794,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	epoch = ++boot_stage.rmcall_gr_submit_epoch;
 	if (!epoch)
 		epoch = ++boot_stage.rmcall_gr_submit_epoch;
+	boot_stage.rmcall_gr_shadow_win0_used =
+		(epoch & 1u) * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS;
+	win0_bank_base = boot_stage.rmcall_gr_shadow_win0_used;
+	shadow_slot_limit = win0_bank_base +
+			    GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS;
 	memcpy(qmd, req->qmd, sizeof(qmd));
 	/*
 	 * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's
@@ -17034,7 +17174,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 					gsp_phys_rmcall_gr_shadow_win0_value() &&
 					boot_stage.rmcall_gr_shadow_win0.pages &&
 					boot_stage.rmcall_gr_shadow_win0_used <
-						GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS;
+						shadow_slot_limit;
 
 				/*
 				 * m86: SASS override. When gsp-phys-gr-override
@@ -17504,24 +17644,37 @@ gr_page_unmapped:
 		gsp_phys_rmcall_gr_gpc_mmu_init(bar0);
 	/*
 	 * m143 DIAGNOSTIC (second-client launch-state divergence): dump the
-	 * exact param page the FE/SM will read (win0 slot 0, the QMD bank-0
-	 * target) and the guest param page at the three JIT parameter slots,
-	 * right before the doorbell. The JIT reads a@0x160/b@0x168/c@0x170
-	 * and n@0x178 (verified with ptxas -arch=sm_75). If the win0 copy is
-	 * correct here but the engine still stores to the wrong relocated
-	 * window, the divergence is engine-side (a stale mapping), not the
-	 * relaunched page content.
+	 * exact param page the FE/SM will read (this launch's win0 bank base
+	 * slot, the QMD bank-0 target) and the guest param page at the three
+	 * JIT parameter slots, right before the doorbell. The JIT reads
+	 * a@0x160/b@0x168/c@0x170 and n@0x178 (verified with ptxas
+	 * -arch=sm_75). If the win0 copy is correct here but the engine still
+	 * stores to the wrong relocated window, the divergence is engine-side
+	 * (a stale mapping), not the relaunched page content.
 	 */
 	if (boot_stage.rmcall_gr_shadow_win0.pages) {
 		const u8 *wp = gsp_phys_chunk_page_va(
-			&boot_stage.rmcall_gr_shadow_win0, 0);
+			&boot_stage.rmcall_gr_shadow_win0, win0_bank_base);
+
+		if (wp) {
+			u32 off;
 
-		if (wp)
-			pr_info("vgsp-gr: DIAG win0[0] 160=%llx 168=%llx 170=%llx 178=%llx\n",
+			pr_info("vgsp-gr: DIAG win0[%u] 160=%llx 168=%llx 170=%llx 178=%llx\n",
+				win0_bank_base,
 				(u64)get_unaligned_le64(wp + 0x160),
 				(u64)get_unaligned_le64(wp + 0x168),
 				(u64)get_unaligned_le64(wp + 0x170),
 				(u64)get_unaligned_le64(wp + 0x178));
+			pr_info("vgsp-gr: win0 param hash pre=0x%016llx\n",
+				gsp_phys_gr_fnv1a64(wp, GSP_PHYS_PAGE_SIZE));
+			for (off = 0x100u; off < 0x200u; off += 0x20u)
+				pr_info("vgsp-gr: win0 page +0x%03x: %016llx %016llx %016llx %016llx\n",
+					off,
+					(u64)get_unaligned_le64(wp + off),
+					(u64)get_unaligned_le64(wp + off + 8),
+					(u64)get_unaligned_le64(wp + off + 16),
+					(u64)get_unaligned_le64(wp + off + 24));
+		}
 	}
 	{
 		u32 pi;
@@ -17533,12 +17686,15 @@ gr_page_unmapped:
 			    !req->pages[pi].hpa)
 				continue;
 			gp = (const u8 *)__va(req->pages[pi].hpa);
+			guest_param_pg = gp;
 			pr_info("vgsp-gr: DIAG guest-param pa=0x%llx 160=%llx 168=%llx 170=%llx 178=%llx\n",
 				req->pages[pi].hpa,
 				(u64)get_unaligned_le64(gp + 0x160),
 				(u64)get_unaligned_le64(gp + 0x168),
 				(u64)get_unaligned_le64(gp + 0x170),
 				(u64)get_unaligned_le64(gp + 0x178));
+			pr_info("vgsp-gr: guest-param hash pre=0x%016llx\n",
+				gsp_phys_gr_fnv1a64(gp, GSP_PHYS_PAGE_SIZE));
 			break;
 		}
 	}
@@ -17969,6 +18125,17 @@ gr_page_unmapped:
 
 	gsp_phys_gr_ok++;
 	/*
+	 * m151: retire the output footprint before the copy-out (and before
+	 * any later launch's copy-in). qmd_rel above is a schedule/fetch ack,
+	 * not a data-path retirement, so the engine's stores into the bounce
+	 * pages can land after it and be lost by an early copy-out; the next
+	 * launch's copy-in would also race that write tail. Success path
+	 * only - the timeout/fault paths never copied out. See
+	 * gsp_phys_gr_output_retire_wait().
+	 */
+	if (req->n_data && boot_stage.rmcall_gr_bounce.pages)
+		gsp_phys_gr_output_retire_wait(req, bounce_csum_in);
+	/*
 	 * m114: the kernel ran against the pre-commit bounce window; copy the
 	 * results back to the guest's real pages (identity HPAs in the
 	 * request). All mapped pages are written back; inputs are unchanged
@@ -18060,6 +18227,23 @@ gr_page_unmapped:
 			ioread32(bar0 + 0x00407060u),
 			ioread32(bar0 + 0x00407064u),
 			ioread32(bar0 + 0x00407068u));
+	/* m151: post-poll page hashes; compare with the pre-doorbell ones. */
+	{
+		const u8 *wp = boot_stage.rmcall_gr_shadow_win0.pages ?
+			gsp_phys_chunk_page_va(
+				&boot_stage.rmcall_gr_shadow_win0,
+				win0_bank_base) : NULL;
+
+		if (wp || guest_param_pg)
+			pr_info("vgsp-gr: win0 param hash post=0x%016llx guest-param hash post=0x%016llx\n",
+				wp ? gsp_phys_gr_fnv1a64(wp,
+							 GSP_PHYS_PAGE_SIZE) :
+				     0ULL,
+				guest_param_pg ?
+					gsp_phys_gr_fnv1a64(guest_param_pg,
+							    GSP_PHYS_PAGE_SIZE) :
+					0ULL);
+	}
 	pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n",
 		req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1,
 		jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok);