vgsp: TSG-sibling mode 30, UVM-stall Patch C, QMD shadow-window ensure + launch diagnostics

3 files changed, 352 insertions, 44 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 60b57f2..4cbbd40 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -650,6 +650,16 @@ bool gsp_phys_boot_requested(void)
  */
 #define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST	29u
 /*
+ * Mode 30 (2026-10-06, experiment): the TSG-sibling split. A clean UNWIRED
+ * golden channel A (captured shape, no ring/USERD/HOBJ wiring) is parented to
+ * the ladder TSG and gets the 9-entry promote plus its own TURING_A (0xc597)
+ * construct, so the first GR init + forced golden save + TSG-wide commit run
+ * on A. V6 stays the execution channel (no swap) and its FWCLIENT promote is
+ * suppressed; the group commit (or the new-channel setup inheritance) must set
+ * V6's instance ENG_CTX pointer and RAMFC ENG_CTX_VALID.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING	30u
+/*
  * Mode-28 golden execution channel shape bits (gsp_phys_rmcall_golden_exec_shape,
  * default 7). Every combination keeps the captured flags 0x00200320 and the
  * FBMEM instance/RAMFC/mthd slices; the bits only choose the client-visible
@@ -763,7 +773,7 @@ bool gsp_phys_boot_requested(void)
  */
 #define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_SYSMEM	(1u << 16)
 #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
-	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST)
+	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING)
 
 /*
  * Work-submit token notifier slot. The default index is
@@ -6889,6 +6899,9 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
 				   const char *what);
 static void gsp_phys_rmcall_pdb_build(void);
 static u64 gsp_phys_rmcall_pde(u64 pa);
+static void gsp_phys_rmcall_ctx_diag_dump(const char *what,
+					  struct gsp_phys_boot_chunk *c);
+static void gsp_phys_rmcall_ctx_diag_decode(const char *tag);
 
 /*
  * Shared-management VAS: install the ladder's page-table window through the
@@ -7614,6 +7627,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
 		return "mode 28: golden execution lane (golden channel with client GPFIFO/USERD, 9-entry promote on it, execution switched to it)";
 	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST:
 		return "mode 29: construct-last golden execution lane (mode 28 minus the TURING_COMPUTE_A object construct on the wired golden channel)";
+	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING:
+		return "mode 30: clean unwired TSG sibling A (promote + TURING_A first-init) + V6 execution";
 	default:
 		return "off (PROMOTE_CTX never sent)";
 	}
@@ -7800,7 +7815,9 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)) {
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING)) {
 		vars = &gsp_phys_rmcall_channel_variant_skipctx;
 		attempts = 1;
 	}
@@ -8259,7 +8276,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) &&
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) &&
 	    gsp_phys_rmcall_engine_is_gr()) {
 		if (gsp_phys_rmcall_fwclient_mask_value() &
 		    GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) {
@@ -8367,7 +8386,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) &&
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) &&
 	    gsp_phys_rmcall_engine_is_gr()) {
 		ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9);
 		ch.internal_flags = cpu_to_le32(
@@ -8467,6 +8488,20 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 				ch.h_va_space = cpu_to_le32(0);
 			}
 		}
+		/*
+		 * Mode 30: clean unwired sibling. Captured flags/memdescs stay;
+		 * the submission wiring is deliberately absent (gpFifoOffset 0,
+		 * captured userd slice, hObjectBuffer 0) and the channel joins
+		 * the ladder TSG so the promote's group commit and A's
+		 * first-init construct are visible to V6. hVASpace=0 per the TSG
+		 * channel rule (kernel_channel.c:429-435).
+		 */
+		if (boot_stage.rmcall_gr_promote ==
+				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) {
+			ch_parent = boot_stage.rmcall_tsg;
+			ch.h_va_space = cpu_to_le32(0);
+			pr_info("vgsp-boot: rm-call: mode 30 GOLDEN SIBLING A: TSG-parented unwired captured shape (TURING_A will construct here for the first GR init + commit)\n");
+		}
 		pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=FBMEM memdescs block=0x%llx (capture-matched apertures)\n",
 			GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
 			GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES, golden_fb);
@@ -8555,14 +8590,31 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) &&
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) &&
 	    gsp_phys_rmcall_engine_is_gr()) {
 		ret = gsp_phys_rmcall_golden9_promote(bar0);
 		if (ret)
 			return ret;
 	}
 
-	/* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */
+	/*
+	 * 6. TURING_A (0xc597) 3D object on the golden channel
+	 * (kernel_graphics.c:2409-2433).
+	 *
+	 * Mode 30 deliberately falls through the skip gate below (it only
+	 * matches modes 28/29 with shape bit5): sibling A keeps its own
+	 * TURING_A construct, and that first GR object construct on A drives
+	 * the first-init + forced golden save (grAlloc_GK104 -> first-init ->
+	 * grInitGoldenCtxImage) whose image and TSG-group commit V6 later
+	 * inherits.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) {
+		pr_info("vgsp-boot: rm-call: mode 30: GOLDEN TURING_A on sibling A skipped by design (m30a: any GR object construct on A blocks RM -110); A is a pure carrier for the 9-entry promote's group commit, the ladder's TURING_COMPUTE_A on V6 provides the instance state\n");
+		return 0;
+	}
 	if ((boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
 	     boot_stage.rmcall_gr_promote ==
@@ -10045,13 +10097,27 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 	if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY ||
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
-	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) &&
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) &&
 	    (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) {
 		pr_err("vgsp-boot: rm-call: mode %u requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
 		       mode);
 		return -EINVAL;
 	}
 	/*
+	 * Mode 30: the sibling A's 9-entry promote carries the group context
+	 * commit; nothing is sent on the V6 channel. A multi-entry promote
+	 * containing MAIN is refused for a SKIP-ctx channel
+	 * (graphics_context.c: bKMDVirtualContext -> NV_ERR_INVALID_ARGUMENT)
+	 * and the single-MAIN exemption needs the pCtxPhysMemDesc the FWCLIENT
+	 * shapes never establish, so a V6 promote would abort stage 3 on 0x1f.
+	 */
+	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) {
+		pr_info("vgsp-boot: rm-call: mode 30: no PROMOTE_CTX on V6; the sibling A carries the group context commit (config_group ppEngCtxDesc), V6 inherits via the TSG commit/channel setup\n");
+		boot_stage.rmcall_gr_promote_step = 1;
+		return 0;
+	}
+	/*
 	 * Mode 28: the 9-entry promote already ran on the golden execution
 	 * channel inside the golden replay (it owns the committed context and
 	 * the forwarded launches); nothing is sent on the V6 channel at all.
@@ -11881,7 +11947,9 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 			     boot_stage.rmcall_gr_promote ==
 					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
 			     boot_stage.rmcall_gr_promote ==
-					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) &&
+					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
+			     boot_stage.rmcall_gr_promote ==
+					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) &&
 			    gsp_phys_rmcall_engine_is_gr()) {
 				pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-%u golden replay failed (%d); the golden construct is atomic and is not retried\n",
 				       boot_stage.rmcall_gr_promote, ret);
@@ -11926,6 +11994,25 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 		return ret;
 
 	/*
+	 * Mode 30 readback point 1: V6's instance/RAMFC right after the channel
+	 * alloc (A's replay contains the 9-entry promote and A's TURING_A
+	 * first-init by now), so A's commit effect can be separated from the
+	 * later BIND/GPFIFO_SCHEDULE. The decode line prints the two decisive
+	 * fields (instance WFI pointer pair + RAMFC ENG_CTX_VALID).
+	 */
+	if (gsp_phys_rmcall_engine_is_gr() &&
+	    boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) {
+		gsp_phys_rmcall_ctx_diag_dump(
+			"INSTANCE (post-A-commit pre-BIND)",
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]);
+		gsp_phys_rmcall_ctx_diag_dump(
+			"RAMFC (post-A-commit pre-BIND)",
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]);
+		gsp_phys_rmcall_ctx_diag_decode("post-A-commit pre-BIND");
+	}
+
+	/*
 	 * Mode 28: the golden execution lane. The golden replay allocated the
 	 * golden channel with a client GPFIFO/USERD and the accepted 9-entry
 	 * context commit; switch the ladder's execution channel to it BEFORE
@@ -13758,6 +13845,50 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page)
 }
 
 /*
+ * Is the 2 MiB window containing `va` marked installed in the engine's table
+ * bookkeeping? Scratch-pool VAs consult rmcall_guest_pt_win_mask (bit = pool
+ * page); every other VA consults the Tier-1 window pool by pdeIndex. Called
+ * with gsp_phys_rm_lock held.
+ */
+static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va)
+{
+	u32 pde_index = (u32)(va >> 21);
+	u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21);
+	u32 i;
+
+	if (pde_index >= base &&
+	    pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES)
+		return !!(boot_stage.rmcall_guest_pt_win_mask &
+			  BIT(pde_index - base));
+	for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++)
+		if (boot_stage.rmcall_gr_win[i].used &&
+		    boot_stage.rmcall_gr_win[i].pde_index == pde_index)
+			return true;
+	return false;
+}
+
+/*
+ * Make the 2 MiB window containing `va` reachable by the engine: scratch-pool
+ * VAs install their pool window through the scratch installer (shared VAS
+ * only; an external VAS keeps the client PDB spine), every other VA installs
+ * its Tier-1 window at its own pdeIndex. Both paths are idempotent; a window
+ * already installed is a bookkeeping hit and sends no RPC. Called with
+ * gsp_phys_rm_lock held.
+ */
+static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va)
+{
+	u32 pde_index = (u32)(va >> 21);
+	u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21);
+	u32 slot;
+
+	if (pde_index >= base &&
+	    pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES)
+		return gsp_phys_rmcall_gr_scratch_win_install_locked(
+			pde_index - base);
+	return gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot);
+}
+
+/*
  * Tier-2 map: consume one never-reused 128 B fetch line from the guest
  * scratch PT pool (PTE0 only) for a ladder-owned shadow page and return the
  * line's base VA. The shadow chunk is allocated lazily; page slot `slot`
@@ -13899,6 +14030,9 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	u8 *qmd_dst_mem = NULL;
 	u32 qmd_rel_off = 0, ch_rel_off = 0;
 	u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0;
+	u32 qmd_win_pde = 0, qmd_win_installed = 0;
+	int qmd_win_ret = 0;
+	bool qmd_win_diag = false;
 	int ret = 0;
 
 	if (out)
@@ -14236,12 +14370,14 @@ gr_page_unmapped:
 	if (req->qmd_hpa) {
 		u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
 		u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1));
+		bool dst_ok = false;
 
 		if (qmd_page &&
 		    !gsp_phys_rmcall_gr_map_va_locked(bar0, qmd_page,
 						      req->qmd_hpa, epoch)) {
 			qmd_dst = qmd_page;
 			qmd_dst_mem = (u8 *)__va(req->qmd_hpa) + qmd_off;
+			dst_ok = true;
 		} else {
 			u64 sh_va = 0;
 
@@ -14253,8 +14389,27 @@ gr_page_unmapped:
 						n_shadow) + qmd_off;
 				n_shadow++;
 				qmd_dst = sh_va + qmd_off;
+				dst_ok = true;
 			}
 		}
+		/*
+		 * The engine must reach the rewritten destination: make sure
+		 * the 2 MiB window containing it is installed in the engine's
+		 * table view (a Tier-1 guest window, or the Tier-2 scratch
+		 * pool window the shadow line lives in). Idempotent; only a
+		 * window not yet installed costs an RPC.
+		 */
+		if (dst_ok) {
+			qmd_win_diag = true;
+			qmd_win_pde = (u32)(qmd_dst >> 21);
+			qmd_win_installed =
+				gsp_phys_rmcall_gr_win_is_installed_locked(
+					qmd_dst) ? 1u : 0u;
+			qmd_win_ret = gsp_phys_rmcall_gr_win_ensure_locked(
+				bar0, qmd_dst);
+			if (!qmd_win_ret)
+				qmd_win_installed = 1u;
+		}
 		pr_info("vgsp-gr: inline-QMD dest va=0x%llx hpa=0x%llx -> 0x%llx (shadow=%u)\n",
 			req->qmd_va, req->qmd_hpa, qmd_dst, n_shadow);
 	}
@@ -14324,6 +14479,10 @@ gr_page_unmapped:
 		req->h_object, req->push_va, req->push_len, req->n_pages,
 		n_shadow, req->qmd_va, sub.slot, words, boot_stage.rmcall_token,
 		qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS);
+	if (qmd_win_diag)
+		pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n",
+			qmd_dst, qmd_win_pde, qmd_win_installed,
+			boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret);
 
 	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS);
 	for (;;) {
@@ -14506,6 +14665,40 @@ static void gsp_phys_rmcall_ctx_diag_dump(const char *what,
 	}
 }
 
+/*
+ * One-line decode of the two fields that decide whether V6's engine context is
+ * live: the instance block's NV_RAMIN_ENGINE_WFI_PTR pair (dword 132 bits
+ * 31:12 = PA[31:12], dword 133 bits 7:0 = PA[39:32]; fifo_gv100.c) and RAMFC
+ * NV_RAMFC_TARGET (dword 43, NV_PPBDMA_TARGET_ENG_CTX_VALID bit 16). Host-view
+ * LE chunks, same as the dump above; used at the mode-30 post-replay and the
+ * stage-4 post-BIND points so A's commit can be told apart from BIND.
+ */
+static void gsp_phys_rmcall_ctx_diag_decode(const char *tag)
+{
+	struct gsp_phys_boot_chunk *inst =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE];
+	struct gsp_phys_boot_chunk *ramfc =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC];
+	const __le32 *iv, *rv;
+	u64 ctx_pa;
+	u32 target;
+	bool valid;
+
+	if (!inst->pages || !ramfc->pages) {
+		pr_info("vgsp-boot: rm-call: V6 ctx (%s): instance/RAMFC chunk absent\n",
+			tag);
+		return;
+	}
+	iv = gsp_phys_chunk_page_va(inst, 0);
+	rv = gsp_phys_chunk_page_va(ramfc, 0);
+	ctx_pa = ((u64)(le32_to_cpu(iv[133]) & 0xffu) << 32) |
+		 (le32_to_cpu(iv[132]) & 0xfffff000u);
+	target = le32_to_cpu(rv[43]);
+	valid = !!(target & (1u << 16));
+	pr_info("vgsp-boot: rm-call: V6 ctx (%s): inst_wfi=0x%llx RAMFC_TARGET=0x%08x ENG_CTX_VALID=%u\n",
+		tag, ctx_pa, target, valid);
+}
+
 static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 {
 	struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
@@ -14616,6 +14809,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]);
 		gsp_phys_rmcall_ctx_diag_dump("RAMFC",
 			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]);
+		gsp_phys_rmcall_ctx_diag_decode("post-BIND");
 	}
 
 	/*
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 47a7b32..2f441a0 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -551,8 +551,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val,
 		return ret;
 
 	if (gsp_phys_rmcall_gr_promote_param < 0 ||
-	    gsp_phys_rmcall_gr_promote_param > 29) {
-		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..29; clamping to 0 (no PROMOTE_CTX)\n",
+	    gsp_phys_rmcall_gr_promote_param > 30) {
+		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..30; clamping to 0 (no PROMOTE_CTX)\n",
 			gsp_phys_rmcall_gr_promote_param);
 		gsp_phys_rmcall_gr_promote_param = 0;
 	}
@@ -571,7 +571,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote,
 		&gsp_phys_rmcall_gr_promote_ops,
 		&gsp_phys_rmcall_gr_promote_param, 0644);
 MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote,
-		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; 26/27: golden9 promote on the V6 execution channel (dead ends); 28: the golden execution lane - the golden channel gets a client GPFIFO ring (VA_GPFIFO), a sysmem host-writable USERD and the ladder TSG as parent, keeps the 9-entry promote, and the ladder switches execution to it (V6 stays unscheduled); 29: mode 28 minus the TURING_COMPUTE_A (0xc5c0) object construct on the client-wired golden channel (every GR object construct there deadlocks RM), i.e. the construct-last golden execution lane; values outside 0..29 clamp to 0 with a warning");
+		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; 26/27: golden9 promote on the V6 execution channel (dead ends); 28: the golden execution lane - the golden channel gets a client GPFIFO ring (VA_GPFIFO), a sysmem host-writable USERD and the ladder TSG as parent, keeps the 9-entry promote, and the ladder switches execution to it (V6 stays unscheduled); 29: mode 28 minus the TURING_COMPUTE_A (0xc5c0) object construct on the client-wired golden channel (every GR object construct there deadlocks RM), i.e. the construct-last golden execution lane; 30: the TSG-sibling split: a clean unwired golden sibling A (captured shape, TSG-parented) runs the 9-entry promote and its own TURING_A first-init while V6 stays the execution channel (no swap) with its FWCLIENT promote suppressed; values outside 0..30 clamp to 0 with a warning");
 
 int gsp_phys_rmcall_gr_promote_value(void)
 {
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 83ca6bd..0605692 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -275,6 +275,13 @@ struct svc_channel {
 				 * counter page can land a moment after the
 				 * first push decode); flushed as soon as the
 				 * slot is registered. 0 = none */
+	u32	pending_rel_inc;	/* 1 = the queued release is an increment
+				 * (add payload to the slot), 0 = absolute */
+	u32	relearn_n;	/* counter-slot re-registrations seen for this
+				 * channel: a healthy GR setup re-learns each
+				 * slot a bounded number of times, while a
+				 * clobbered slot loops forever (storm
+				 * detector) */
 };
 
 /*
@@ -516,6 +523,9 @@ struct sema_svc {
 						 * deliverable exact slot */
 	u64			counter_rel_applied;	/* releases to registered counters */
 	u64			counter_rel_dbg;	/* rel-apply diagnostic budget */
+	u64			counter_noslot_dbg;	/* "no slot registered" budget,
+						 * separate so it cannot hide the
+						 * written/skip lines */
 	u64			counter_pub_dbg;	/* counter-publish diagnostic budget */
 	u64			counter_pub_calls;	/* publishes that found a registered counter */
 	u64			gr_rel_applied;	/* release writes into the GR counter page */
@@ -4238,14 +4248,20 @@ static inline bool svc_counter_va(u64 va)
  * through the sema_counter parameter from the client's own pagemap. Writing
  * the payload there is what lets the client's wait observe the release
  * regardless of the VA map. Forward-only, so a stale copy of an
- * already-completed push cannot rewind the counter.
+ * already-completed push cannot rewind the counter. `inc` selects UVM's
+ * increment-release semantics (write cur + payload) instead of the absolute
+ * semantics (write payload); GR tracking-semaphore pushes release an
+ * increment, and an absolute-only guard drops every increment after the
+ * first one because payload == cur.
  */
 static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
-				      u64 sem_va, u32 payload)
+				      u64 sem_va, u32 payload, bool inc)
 {
 	unsigned long flags;
 	u64 pa = 0;
 	u32 cur = 0;
+	u32 nv;
+	bool skip;
 	int i;
 	u32 dbg = 0;
 
@@ -4284,9 +4300,12 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 				if (svc.chan[i].valid &&
 				    svc.chan[i].h_object == h) {
 					if (payload >
-					    svc.chan[i].pending_rel_pay)
+					    svc.chan[i].pending_rel_pay) {
 						svc.chan[i].pending_rel_pay =
 							payload;
+						svc.chan[i].pending_rel_inc =
+							inc ? 1 : 0;
+					}
 					break;
 				}
 			}
@@ -4296,8 +4315,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 				pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> slot pending\n",
 					h, sem_va, payload);
 			}
-		} else if (gsp_sema_dbg && svc.counter_rel_dbg < 96) {
-			svc.counter_rel_dbg++;
+		} else if (gsp_sema_dbg && svc.counter_noslot_dbg < 16) {
+			svc.counter_noslot_dbg++;
 			pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> no slot registered\n",
 				h, sem_va, payload);
 		}
@@ -4312,7 +4331,17 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 		}
 		return;
 	}
-	if (payload <= cur || payload - cur > 0x100000) {
+	/*
+	 * Absolute releases must be forward-only (a stale copy of a completed
+	 * push must not rewind the counter); increment releases add the
+	 * payload, like the generic LAUNCH path does. The jump guard keeps a
+	 * bogus payload from teleporting the counter past the client's wait.
+	 */
+	if (inc)
+		skip = payload > 0x100000;
+	else
+		skip = payload <= cur || payload - cur > 0x100000;
+	if (skip) {
 		if (gsp_sema_dbg && svc.counter_rel_dbg < 32) {
 			svc.counter_rel_dbg++;
 			pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> skip\n",
@@ -4320,8 +4349,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 		}
 		return;
 	}
-	if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &payload,
-			 sizeof(payload)))
+	nv = inc ? cur + payload : payload;
+	if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv)))
 		return;
 	svc.releases_applied++;
 	svc.counter_rel_applied++;
@@ -4336,34 +4365,74 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 }
 
 /*
+ * Clear a channel's queued release in the authoritative array. The caller of
+ * svc_counter_flush_pending() usually works on a pass snapshot, so clearing
+ * only the snapshot would let the next pass flush the same payload again -
+ * harmless for an absolute release (its guard drops it), but an increment
+ * would advance the slot twice.
+ */
+static void svc_counter_pending_clear(u32 h)
+{
+	unsigned long flags;
+	int i;
+
+	if (!h)
+		return;
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
+			svc.chan[i].pending_rel_pay = 0;
+			svc.chan[i].pending_rel_inc = 0;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+}
+
+/*
  * Deliver a release that was decoded before the channel's counter slot was
  * registered (see pending_rel_pay). Called whenever a slot becomes known and
  * from the per-pass publish path, so a queued payload lands as soon as the
- * injected pin is visible to the service.
+ * injected pin is visible to the service. An increment-queued payload adds to
+ * the slot (UVM INC semantics), an absolute one writes its value.
  */
 static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
 				      struct svc_channel *ch)
 {
 	u32 payload = ch->pending_rel_pay;
+	bool inc = ch->pending_rel_inc;
 	u32 cur = 0;
+	u32 nv;
 
 	if (!payload || !ch->counter_pa)
 		return;
 	if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur,
 			sizeof(cur)))
 		return;
-	if (payload <= cur || payload - cur > 0x100000) {
+	if (inc) {
+		if (payload > 0x100000) {
+			ch->pending_rel_pay = 0;
+			ch->pending_rel_inc = 0;
+			svc_counter_pending_clear(ch->h_object);
+			return;
+		}
+	} else if (payload <= cur || payload - cur > 0x100000) {
 		ch->pending_rel_pay = 0;
+		ch->pending_rel_inc = 0;
+		svc_counter_pending_clear(ch->h_object);
 		if (cur > ch->counter_last)
 			ch->counter_last = cur;
 		return;
 	}
-	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &payload,
-			 sizeof(payload)))
+	nv = inc ? cur + payload : payload;
+	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &nv,
+			 sizeof(nv)))
 		return;
 	ch->pending_rel_pay = 0;
-	if (payload > ch->counter_last)
-		ch->counter_last = payload;
+	ch->pending_rel_inc = 0;
+	svc_counter_pending_clear(ch->h_object);
+	if (nv > ch->counter_last)
+		ch->counter_last = nv;
 	svc.releases_applied++;
 	svc.counter_rel_applied++;
 	svc.gr_rel_applied++;
@@ -4494,33 +4563,61 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
 			return;
 		derived = true;
 	}
-	ch->counter_pa = pa & ~0x3ULL;
+	pa &= ~0x3ULL;
 	/*
-	 * Persist the registration into the authoritative channel array
-	 * immediately, under the lock, instead of relying on the pump's
-	 * end-of-pass write-back: the pump works on a snapshot and several
-	 * early `continue`s (USERD not yet resolvable, ring not yet
-	 * readable) skip that write-back. When that happened every pass
-	 * re-registered the same slot - the "counter auto" line flooded the
-	 * log ring (1.17M lines, noclaim27) and, worse, the publish path
-	 * never saw a registered counter (`pub_calls=0`), so the client's
-	 * tracker semaphores were never advanced and cudaMalloc hung. The
-	 * authoritative store makes the registration durable and the log
-	 * one-shot.
+	 * Never install a slot another live channel already owns: the
+	 * rank-derived formula is evaluated while channels are still
+	 * registering and can name a slot a lower rank already registered
+	 * (observed: rank3 -> rank0's slot ...ec0). The collision makes every
+	 * pass flip the slot (svc_counter_learn writes the exact slot, the
+	 * pass write-back restores the guess) and the client's tracker never
+	 * completes; leave counter_pa zero for a later pass instead.
 	 */
 	{
 		unsigned long flags;
 		int j;
+		bool collision = false;
 
 		spin_lock_irqsave(&svc.chan_lock, flags);
 		for (j = 0; j < SVC_MAX_CHANNELS; j++) {
 			if (svc.chan[j].valid &&
-			    svc.chan[j].h_object == ch->h_object) {
-				svc.chan[j].counter_pa = ch->counter_pa;
+			    svc.chan[j].h_object != ch->h_object &&
+			    svc.chan[j].counter_pa == pa) {
+				collision = true;
 				break;
 			}
 		}
+		if (!collision) {
+			/*
+			 * Persist the registration into the authoritative
+			 * channel array immediately, under the lock, instead
+			 * of relying on the pump's end-of-pass write-back:
+			 * the pump works on a snapshot and several early
+			 * `continue`s (USERD not yet resolvable, ring not
+			 * yet readable) skip that write-back. When that
+			 * happened every pass re-registered the same slot
+			 * (1.17M log lines, noclaim27) and, worse, the
+			 * publish path never saw a registered counter
+			 * (`pub_calls=0`), so the client's tracker
+			 * semaphores were never advanced and cudaMalloc
+			 * hung. The authoritative store makes the
+			 * registration durable and the log one-shot.
+			 */
+			ch->counter_pa = pa;
+			for (j = 0; j < SVC_MAX_CHANNELS; j++) {
+				if (svc.chan[j].valid &&
+				    svc.chan[j].h_object == ch->h_object) {
+					svc.chan[j].counter_pa = pa;
+					break;
+				}
+			}
+		}
 		spin_unlock_irqrestore(&svc.chan_lock, flags);
+		if (collision) {
+			pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n",
+					    ch->h_object, ch->gr_index, pa);
+			return;
+		}
 	}
 	if (derived)
 		pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
@@ -4570,6 +4667,11 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
 		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
 			if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) {
 				svc.chan[i].counter_pa = pa & ~0x3ULL;
+				svc.chan[i].relearn_n++;
+				if (svc.chan[i].relearn_n == 64)
+					pr_warn_ratelimited("sema-chan: counter re-learn storm h=0x%x n=%u (slot clobbered; UVM will stall)\n",
+							    h,
+							    svc.chan[i].relearn_n);
 				if (derived)
 					pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n",
 						h, pa & ~0x3ULL, sem_va);
@@ -4600,7 +4702,8 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu,
 
 	if (ch->h_object && svc_counter_va(va)) {
 		svc_counter_learn(vgpu, ch->h_object, va);
-		svc_counter_release_apply(vgpu, ch->h_object, va, payload);
+		svc_counter_release_apply(vgpu, ch->h_object, va, payload,
+					  op == HOST_SEM_OP_REDUCTION);
 		return false;
 	}
 	if (!svc_sem_read(vgpu, va, &cur) && payload > cur) {
@@ -4623,7 +4726,8 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu,
 
 	if (ch->h_object && svc_counter_va(va)) {
 		svc_counter_learn(vgpu, ch->h_object, va);
-		svc_counter_release_apply(vgpu, ch->h_object, va, payload);
+		svc_counter_release_apply(vgpu, ch->h_object, va, payload,
+					  false);
 		return false;
 	}
 	if (svc_sem_read(vgpu, va, &cur)) {
@@ -4668,7 +4772,8 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
 	}
 	if (ch->engine_type == 0)
 		svc_counter_learn(vgpu, ch->h_object, sem_va);
-	svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload);
+	svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
+				  inc && inc_apply);
 	return false;
 }
 
@@ -11226,9 +11331,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				 * snapshot: without this the next pass re-derives
 				 * it (with a different live GR count during
 				 * teardown) and the publishes target the wrong
-				 * slot.
+				 * slot. Never let the pass-start snapshot revert
+				 * a slot learned mid-pass (svc_counter_learn) or
+				 * registered from the injected pin: the snapshot
+				 * may still hold the rank-derived autoreg guess
+				 * (or 0), which made the registration
+				 * non-durable (24,834 re-learn lines in one
+				 * stalled run vs <=35 per channel in passing
+				 * runs, 2026-10-06). Only fill a missing slot; a
+				 * live authoritative value wins.
 				 */
-				svc.chan[k].counter_pa = ch->counter_pa;
+				if (!svc.chan[k].counter_pa)
+					svc.chan[k].counter_pa = ch->counter_pa;
 				break;
 			}
 		}