r434 wedge instrumentation + CWD-slot path: post-retire QMD64 dump (hardware-only w50/w51 INNER get/put = the SKED's CTA dispatch counters, which the submission-time dump cannot show) so the healthy-vs-wedged diff proves whether the grid was dispatched and truncated; post-OK qmdwin reads the launch's own bank; gsp_phys_rmcall_gr_cwd_slot >= 0 emits the CUDA driver's CWD-slot PRI write (SKED_DEBUG_1 ENABLE_CWD_SLOT_COUNT via falcon scratch) + SET_CWD_SLOT_COUNT in the init preamble for the mission's slot-count A/B (-1 = off)

3 files changed, 114 insertions, 2 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index f4e558f..1b8deae 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -16309,6 +16309,19 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
 #define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT			1536u
 #define GSP_PHYS_RMCALL_GR_CWD_LEVELS			64u
 #define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT			24u
+/*
+ * r434 CWD-slot-count path: the falcon PRI write (CUDA's
+ * turingFalconPRIWriteMasked: flag/data/mask through SET_MME_SHADOW_SCRATCH
+ * 0/1/2, then SET_FALCON04 = register offset) and the gated method. Used only
+ * when gsp_phys_rmcall_gr_cwd_slot >= 0.
+ */
+#define GSP_PHYS_RMCALL_GR_MME_SCRATCH0			0x3400u
+#define GSP_PHYS_RMCALL_GR_MME_SCRATCH1			0x3404u
+#define GSP_PHYS_RMCALL_GR_MME_SCRATCH2			0x3408u
+#define GSP_PHYS_RMCALL_GR_SET_FALCON04			0x0510u
+#define GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT		0x02b0u
+#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1		0x00407040u
+#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u
 /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) +
  * the optional QMD-virt pair(2). */
 #define GSP_PHYS_RMCALL_GR_METHOD_WORDS		512u
@@ -16529,6 +16542,29 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p)
 				    GSP_PHYS_RMCALL_GR_MAX_SM_COUNT);
 	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL,
 				    0u);
+	/*
+	 * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot >= 0): masked
+	 * PRI write of SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (bit 11) through the
+	 * falcon scratch registers, then the gated SET_CWD_SLOT_COUNT method -
+	 * the CUDA driver's own turingLaunchSetCwdSlotCount sequence. -1 (the
+	 * default) emits nothing, so the historical preamble is unchanged.
+	 */
+	if (gsp_phys_rmcall_gr_cwd_slot_value() >= 0) {
+		p = gsp_phys_rmcall_push_gr(
+			p, GSP_PHYS_RMCALL_GR_MME_SCRATCH0, 0u);
+		p = gsp_phys_rmcall_push_gr(
+			p, GSP_PHYS_RMCALL_GR_MME_SCRATCH1,
+			GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT);
+		p = gsp_phys_rmcall_push_gr(
+			p, GSP_PHYS_RMCALL_GR_MME_SCRATCH2,
+			GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT);
+		p = gsp_phys_rmcall_push_gr(
+			p, GSP_PHYS_RMCALL_GR_SET_FALCON04,
+			GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1);
+		p = gsp_phys_rmcall_push_gr(
+			p, GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT,
+			(u32)gsp_phys_rmcall_gr_cwd_slot_value());
+	}
 	return p;
 }
 
@@ -17806,6 +17842,32 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req,
 	return changed;
 }
 
+/*
+ * r434: post-launch deposit dump (hardware-only words). The r430 QMD64 dump
+ * runs at submission, before the engine touches the deposit, so the
+ * hardware-only fields (w50 INNER_GET, w51 INNER_PUT, w52 span-list head,
+ * w53 next-QMD) are always zero there. Reading the same page AFTER the
+ * retire wait shows what the SKED actually consumed: w50/w51 are the CTA
+ * (INNER) get/put counters, so a wedged launch whose grid "completed" with
+ * zero stores is distinguishable from a genuinely dispatched grid by
+ * whether INNER_PUT advanced to the CTA count. One line per launch, gated
+ * by the skedrec param.
+ */
+static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag)
+{
+	char qb[64 * 9 + 1];
+	int qo = 0;
+	u32 i;
+
+	if (!deposit || !gsp_phys_rmcall_gr_skedrec_value())
+		return;
+	for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10;
+	     i++)
+		qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ",
+				get_unaligned_le32(deposit + i * 4));
+	pr_info("vgsp-gr: QMD64-POST %s %s\n", tag, qb);
+}
+
 static noinline u32
 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req,
 			       const u32 *csum_in, u32 *n_tracked)
@@ -20738,6 +20800,14 @@ gr_page_unmapped:
 							   &n_tracked);
 
 		/*
+		 * r434: post-launch hardware-only QMD dump (w50/w51 INNER
+		 * get/put = the SKED's CTA dispatch counters) so the
+		 * healthy-vs-wedged diff has the fields the submission-time
+		 * dump cannot show.
+		 */
+		gsp_phys_gr_qmd64_post(qmd_dst_mem, "retire");
+
+		/*
 		 * m179: the zero-write test is specific to launches with a
 		 * non-empty TRACKED data set. A launch whose data entries do
 		 * not resolve to any bounce page legitimately reports zero
@@ -21106,6 +21176,8 @@ gr_page_unmapped:
 				}
 				wrote = gsp_phys_gr_output_retire_wait(
 					req, bounce_csum_in, &n_tracked);
+				gsp_phys_gr_qmd64_post(qmd_dst_mem,
+						       "retry-retire");
 				pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n",
 					nw_attempts, wrote);
 				/* r410: the quiet-window verdict. */
@@ -21683,8 +21755,20 @@ gr_page_unmapped:
 		jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok);
 	/* m128: post-success qmdwin words so a probe that stores a value into
 	 * the qmdwin page (e.g. p6's constant-read dump at w48) is readable
-	 * from the success log (the post-mortem only prints on timeout). */
-	if (boot_stage.rmcall_gr_qmdwin.pages) {
+	 * from the success log (the post-mortem only prints on timeout).
+	 * r434: read the launch's OWN bank (qmd_dst_mem) when available -
+	 * the old bank-0-only read showed a stale page for odd epochs. */
+	if (qmd_dst_mem) {
+		const u32 *qw = (const u32 *)qmd_dst_mem;
+
+		pr_info("vgsp-gr: post-OK qmdwin w48..59=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
+			get_unaligned_le32(qw + 48), get_unaligned_le32(qw + 49),
+			get_unaligned_le32(qw + 50), get_unaligned_le32(qw + 51),
+			get_unaligned_le32(qw + 52), get_unaligned_le32(qw + 53),
+			get_unaligned_le32(qw + 54), get_unaligned_le32(qw + 55),
+			get_unaligned_le32(qw + 56), get_unaligned_le32(qw + 57),
+			get_unaligned_le32(qw + 58), get_unaligned_le32(qw + 59));
+	} else if (boot_stage.rmcall_gr_qmdwin.pages) {
 		const u32 *qw = (const u32 *)gsp_phys_chunk_page_va(
 			&boot_stage.rmcall_gr_qmdwin, 0);
 
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 4c0fd37..eadb4c2 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -664,6 +664,29 @@ int gsp_phys_rmcall_gr_pcas_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_value);
 
 /*
+ * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot; default -1 = off).
+ * On Volta+ the SET_CWD_SLOT_COUNT method is gated by the falcon PRI bit
+ * NV_PGRAPH_PRI_SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (the CUDA driver's
+ * turingLaunchSetCwdSlotCount: a masked PRI write via SET_MME_SHADOW_SCRATCH
+ * 0/1/2 + SET_FALCON04, then the method itself). When >= 0 the init preamble
+ * emits that PRI write (ENABLE_CWD_SLOT_COUNT=1) followed by
+ * SET_CWD_SLOT_COUNT with the given value, so the wedge's "grid truncated to
+ * ~4 CTAs" signature can be A/B'd against the CWD slot-count watermark.
+ * Read per launch; -1 restores the historical preamble byte-for-byte.
+ */
+static int gsp_phys_rmcall_gr_cwd_slot_param = -1;
+module_param_named(gsp_phys_rmcall_gr_cwd_slot,
+		   gsp_phys_rmcall_gr_cwd_slot_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_cwd_slot,
+		 "r434: enable the CWD slot count PRI (SKED_DEBUG_1 bit11) and emit SET_CWD_SLOT_COUNT with this value in the GR init preamble (-1 = off, default)");
+
+int gsp_phys_rmcall_gr_cwd_slot_value(void)
+{
+	return gsp_phys_rmcall_gr_cwd_slot_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_cwd_slot_value);
+
+/*
  * Two-step PCAS gate (default 0). With the explicit PCAS pair on, setting
  * this marks the QMD's REQUIRE_SCHEDULING_PCAS (w4 bit12) and emits the copy
  * PCAS (INVALIDATE only) followed by a schedule-only PCAS (SCHEDULE), the
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 8158b46..90783cb 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -393,6 +393,11 @@ int gsp_phys_gr_icache_inv_value(void);
  * before writing the release semaphore (default 0 = guest's FE_NONE). */
 int gsp_phys_gr_rel_membar_value(void);
 
+/* r434: enable the CWD slot count PRI (SKED_DEBUG_1 bit11) and emit
+ * SET_CWD_SLOT_COUNT with this value in the GR init preamble
+ * (-1 = off, default). */
+int gsp_phys_rmcall_gr_cwd_slot_value(void);
+
 /* m179: dedicated re-execution budget for a data-bearing GR launch whose
  * retirement wait saw zero engine writes, independent of the timeout ladder's
  * attempt counter (default 3, clamped [0,4]). */