r425 wedge diagnostics + init-once A/B: SKED/FECS/TPC/CWD register witnesses (healthy vs wedged baselines in-log), prev-out late-store witness, gsp_phys_rmcall_gr_init_once (emit the compute init preamble once per boot vs per launch)

1 files changed, 119 insertions, 1 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index cf5061d..9cf2072 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -16347,6 +16347,50 @@ bool gsp_phys_gr_quiet_hold_active(void)
 }
 EXPORT_SYMBOL_GPL(gsp_phys_gr_quiet_hold_active);
 
+/*
+ * r425 experiment (the second-launch wedge): the stock guest driver emits the
+ * per-channel compute init block ONCE at channel creation (turing_init.c),
+ * long before any launch, while this backend re-emits it as the preamble of
+ * every forwarded launch. The wedge strikes the SECOND launch of a channel, so
+ * test whether the re-emission is the trigger: with gsp_phys_rmcall_gr_init_once
+ * set, the init preamble is emitted only on the first forwarded launch after
+ * the module load (the engine's compute-class/CWD state persists in the
+ * context; every launch still carries its QMD + report semaphore). 0 keeps the
+ * historical per-launch emission (default; runtime-writable A/B).
+ */
+static bool gsp_phys_rmcall_gr_init_once_param;
+module_param_named(gsp_phys_rmcall_gr_init_once, gsp_phys_rmcall_gr_init_once_param,
+		   bool, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_init_once,
+		 "r425: emit the GR compute init preamble only on the first forwarded launch (0 = every launch, default)");
+static bool gsp_phys_rmcall_gr_init_emitted;
+
+/*
+ * r425 stale-QMD witness: the first engine-changed page of the LAST completed
+ * launch (bounce index + post-copy-out checksum). At a zero-write accept the
+ * page is re-scanned; a change proves a late store (a stuck/queued grid or a
+ * stale-QMD execution) landed after its launch was shipped.
+ */
+static u32 gsp_phys_gr_prev_out_bi;
+static u32 gsp_phys_gr_prev_out_csum;
+static bool gsp_phys_gr_prev_out_valid;
+
+/* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged
+ * comparison baseline lives in the same log). */
+static void gsp_phys_gr_sked_wit(void __iomem *bar0, const char *tag)
+{
+	if (!bar0)
+		return;
+	pr_info("vgsp-gr: SKED-WIT %s cwd0=%08x cwd1=%08x cwd2=%08x cwd3=%08x info1=%08x act=%08x grfifo=%08x fecs=%08x new=%08x ctxsw1=%08x qmdvirt0=%08x tpcm0=%08x\n",
+		tag,
+		readl(bar0 + 0x00407060u), readl(bar0 + 0x00407064u),
+		readl(bar0 + 0x00407068u), readl(bar0 + 0x0040706cu),
+		readl(bar0 + 0x00407028u), readl(bar0 + 0x00407054u),
+		readl(bar0 + 0x00400504u), readl(bar0 + 0x00409b00u),
+		readl(bar0 + 0x00409b04u), readl(bar0 + 0x00409400u),
+		readl(bar0 + 0x0040a500u), readl(bar0 + 0x0040a680u));
+}
+
 /* m86: program-page override blob (see gsp-phys-gr-override). */
 static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE];
 static u32 gsp_phys_gr_override_len;
@@ -16476,7 +16520,17 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 	int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value();
 	int two_step = gsp_phys_rmcall_gr_pcas_two_step_value();
 
-	p = gsp_phys_rmcall_build_gr_init(p);
+	/*
+	 * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble
+	 * is emitted only on the first forwarded launch (the stock driver's
+	 * per-channel shape); subsequent launches start at the inline-QMD
+	 * window.
+	 */
+	if (!READ_ONCE(gsp_phys_rmcall_gr_init_once_param) ||
+	    !READ_ONCE(gsp_phys_rmcall_gr_init_emitted)) {
+		p = gsp_phys_rmcall_build_gr_init(p);
+		WRITE_ONCE(gsp_phys_rmcall_gr_init_emitted, true);
+	}
 	if (qmdvirt)
 		p = gsp_phys_rmcall_push_gr(p,
 				GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL,
@@ -18142,6 +18196,8 @@ gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us,
 		w0, want_qmd, w1,
 		(u32)ktime_to_us(ktime_get()) - since_us,
 		usr_gpget, sub->gpget, sub->gpput);
+	/* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */
+	gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll");
 }
 
 /*
@@ -21008,6 +21064,34 @@ gr_page_unmapped:
 					readl(bar0 + 0x00409008u),
 					readl(bar0 + 0x00409b00u),
 					readl(bar0 + 0x00409b04u));
+			/* r425: the wedged-launch SKED/FECS baseline. */
+			gsp_phys_gr_sked_wit(bar0, "accept");
+			/*
+			 * r425 stale-store witness: re-scan the last completed
+			 * launch's first engine-changed page. A change since its
+			 * copy-out proves a late store landed after the launch
+			 * was shipped (a stuck/queued grid or a stale-QMD
+			 * execution), the mechanism behind the "1 page" heals.
+			 */
+			if (gsp_phys_gr_prev_out_valid &&
+			    bounce_staged_hpa[gsp_phys_gr_prev_out_bi] == 0 &&
+			    gsp_phys_gr_data_chunk()->pages) {
+				const u8 *pp = gsp_phys_chunk_page_va(
+					gsp_phys_gr_data_chunk(),
+					gsp_phys_gr_prev_out_bi);
+				u32 pc = 0, pi;
+
+				if (pp) {
+					for (pi = 0; pi < GSP_PHYS_PAGE_SIZE;
+					     pi += 4)
+						pc += get_unaligned_le32(pp + pi);
+					if (pc != gsp_phys_gr_prev_out_csum)
+						pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u csum %08x -> %08x (late store after its launch shipped)\n",
+							gsp_phys_gr_prev_out_bi,
+							gsp_phys_gr_prev_out_csum,
+							pc);
+				}
+			}
 			/*
 			 * (c) Late-store check: a store ramp that retires after
 			 * the host's window is indistinguishable from a lost
@@ -21223,6 +21307,16 @@ gr_page_unmapped:
 				}
 				if (changed < 8)
 					chg[changed] = bi;
+				/*
+				 * r425: remember the first engine-changed page
+				 * (post-copy-out content) for the next zero-write
+				 * accept's late-store witness.
+				 */
+				if (changed == 0) {
+					gsp_phys_gr_prev_out_bi = bi;
+					gsp_phys_gr_prev_out_csum = csum;
+					gsp_phys_gr_prev_out_valid = true;
+				}
 				changed++;
 			}
 			/*
@@ -23536,6 +23630,30 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused)
 	sked_cwd1 = gsp_phys_reg_probe_line(m, bar0,
 					    "SKED_DEBUG_CWD_TABLE_1",
 					    0x00407064u);
+	/* r425: the wedge diagnostics - the rest of the SKED CWD table, the
+	 * per-8-group GR TPC masks (a zero compute mask would dispatch no
+	 * TPCs), and the FE QMD-virtualization control for VEID 0. */
+	(void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_2",
+				      0x00407068u);
+	(void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_3",
+				      0x0040706cu);
+	(void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_INFO_1",
+				      0x00407028u);
+	{
+		u32 gi;
+
+		for (gi = 0; gi < 8u; gi++) {
+			gsp_phys_reg_probe_line(m, bar0,
+				"FE_TPCM_GR_TPC_MASK",
+				0x0040a680u + gi * 4u);
+			gsp_phys_reg_probe_line(m, bar0,
+				"FE_LAST_GR_TPC_MASK",
+				0x0040a6a0u + gi * 4u);
+			gsp_phys_reg_probe_line(m, bar0,
+				"FE_CWD_OWN_TPC_MASK",
+				0x0040a6c0u + gi * 4u);
+		}
+	}
 	fecs_irq = gsp_phys_reg_probe_line(m, bar0, "FECS_IRQSTAT",
 					   0x00409008u);
 	fecs_cpuctl = gsp_phys_reg_probe_line(m, bar0, "FECS_CPUCTL",