vgsp: golden-exec shape bisection + fwclient per-entry promotion + pool-VA stall fix

- Mode 28 golden execution lane behind gsp_phys_rmcall_gr_promote=28 with a
  shape bitmask (gsp_phys_rmcall_golden_exec_shape: ring / entries=8 / sysmem
  USERD / USERD size / hObjectBuffer=VAS / skip replay TURING_A) and a USERD
  control-offset helper. Finding: the RM-managed golden channel hangs any GR
  object construct (-110, no assert) as soon as client submission wiring
  (ring or sysmem USERD) is attached, so only the captured server-owned shape
  constructs; the machinery is kept for future wiring ideas.
- fwclient mask bit 11 (0x800): send each non-MAIN FWCLIENT promote entry as
  its own one-entry control and continue past refusals. V6 acceptance matrix:
  PM, PATCH, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP accepted,
  FECS_EVENT refused 0x1f.
- fwclient mask bit 12 (0x1000): golden9 VA-only global bookings (BUNDLE_CB,
  PAGEPOOL, ATTRIBUTE_CB, RTV_CB_GLOBAL) accepted on V6; bits 13/14
  (0x2000/0x4000): FECS PA-only (refused 0x1f) and FECS VA-only (accepted).
- gsp_phys_rmcall_srv_main_ignore: ignore a server-reported MAIN in the
  srv-ctx commit and force the client INITIALIZE_CTX + VA-only fallback.
- gsp_sema_service: pool semaphores never fall back to the learned map; queue
  the absolute release via svc_pending_add until svc_pool_pa_resolve
  succeeds, and judge pool slots through the verified resolver in
  svc_apply_pending and the release decode (UVM registration stall fix).

1 files changed, 384 insertions, 23 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 040428c..3383b35 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -618,6 +618,38 @@ bool gsp_phys_boot_requested(void)
  */
 #define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY	27u
 /*
+ * Mode 28 (2026-10-06, experiment): the golden execution lane. Mode 25's
+ * captured golden construct (VAS binding, FBMEM instance/RAMFC/mthd slices,
+ * single atomic 9-entry GPU_PROMOTE_CTX on the golden channel), but the
+ * golden channel itself gets a client-visible submission path: the ladder's
+ * GPFIFO ring VA (gpFifoOffset) and the ladder's sysmem USERD control page
+ * (host GP_PUT writes reach the hardware word, the V6 shape). The channel
+ * stays device-parented like the captured golden construct (a golden9
+ * promote on a TSG channel poisons the next GR object construct: modes 27/28
+ * first run both died -110 there), and the real nv-push path schedules
+ * device-parented channels with BIND + channel-level GPFIFO_SCHEDULE alone.
+ * The ladder swaps boot_stage.rmcall_channel to the golden handle before the
+ * 0xc5c0 engine-object alloc, so stage 4's BIND/GPFIFO_SCHEDULE/token
+ * sequence, the GR sem check and every forwarded publish run on the golden
+ * channel; V6 stays allocated but is never bound or scheduled, and its
+ * FWCLIENT promote is skipped. The hypothesis: the golden channel owns the
+ * complete committed context, so the engine will finally dispatch the
+ * forwarded QMD.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC		28u
+/*
+ * Mode-28 golden execution channel shape bits (gsp_phys_rmcall_golden_exec_shape,
+ * default 7). Every combination keeps the captured flags 0x00200320 and the
+ * FBMEM instance/RAMFC/mthd slices; the bits only choose the client-visible
+ * submission path, so a deadlock can be bisected without a rebuild.
+ */
+#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_RING		BIT(0)
+#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_ENTRIES8	BIT(1)
+#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SYSMEM	BIT(2)
+#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SIZE200	BIT(3)
+#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_HOBJ_VAS	BIT(4)
+#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_SKIP_TURING_A	BIT(5)
+/*
  * Existing fwclient mask bit 8: mode-25 gate for the captured VAS
  * COPY_SERVER_RESERVED_PDES control. The captured control copies three
  * server-RM-internal page-table levels the ladder cannot source safely, so
@@ -643,8 +675,34 @@ bool gsp_phys_boot_requested(void)
  * for comparison.
  */
 #define GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS	(1u << 10)
+/*
+ * Mode-25 bisection bit 11: send each built non-MAIN FWCLIENT entry as its
+ * own one-entry GPU_PROMOTE_CTX control instead of one batched control, so a
+ * single refused entry cannot refuse the whole set (m2b/m2f: the batched
+ * five-entry control returned 0x1f as a whole; individual acceptance was
+ * never tested).
+ */
+#define GSP_PHYS_RMCALL_FWCLIENT_MASK_INDIVIDUAL	(1u << 11)
+/*
+ * Mode-25 bisection bit 12: additionally promote the golden9 VA-only global
+ * bookings (bufferIds 3/4/5/6 = BUNDLE_CB, PAGEPOOL, ATTRIBUTE_CB,
+ * RTV_CB_GLOBAL) on the ladder channel as one-entry controls. The golden9
+ * context carries them; the FWCLIENT set does not, so the V6 context may be
+ * missing the GR engine's code/attribute data entirely.
+ */
+#define GSP_PHYS_RMCALL_FWCLIENT_MASK_GOLDEN9_GLOBALS	(1u << 12)
+/*
+ * Mode-25 bisection bits 13/14: the FECS_EVENT entry is refused 0x1f on the
+ * V6 channel in its combined PA+VA form (the same form the golden channel
+ * accepts). Try its halves separately: bit 13 = PA-only init entry
+ * (bInitialize=1 bNonmapped=1, like the accepted PM entry), bit 14 = VA-only
+ * booking entry (bInitialize=0, like the accepted globals). Both booked
+ * together may complete the FECS state the combined control cannot.
+ */
+#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY	(1u << 13)
+#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY	(1u << 14)
 #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
-	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY)
+	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC)
 
 /*
  * Work-submit token notifier slot. The default index is
@@ -946,6 +1004,15 @@ bool gsp_phys_boot_requested(void)
 	      (1u << 21))
 
 /*
+ * Mode-28 alternative golden flags (unused after the m3b first run kept the
+ * captured word): the captured golden word with the USERD placement bits
+ * clear, for experiments where the hardware control block must sit at
+ * userd+0 (publish offset 0x8c).
+ */
+#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN_EXEC \
+	(u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED)
+
+/*
  * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as
  * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc
  * RPC never replied, every later RPC timed out, and the fail-safe held the
@@ -1014,6 +1081,9 @@ enum gsp_phys_rmcall_buf {
 #define GSP_PHYS_RMCALL_GPFIFO_ENTRIES	8u
 #define GSP_PHYS_RMCALL_USERD_GPPUT_OFF	0x8cu	/* Nvc46fControl GPPut (clc46f.h:57) */
 #define GSP_PHYS_RMCALL_USERD_GPGET_OFF	0x88u	/* Nvc46fControl GPGet (clc46f.h:56) */
+/* Captured golden flags pin USERD_INDEX_VALUE=3, so the control block (and
+ * the GP_PUT/GP_GET words inside it) sits 0x600 into the USERD chunk. */
+#define GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED	0x600u
 #define GSP_PHYS_RMCALL_SEM_PAYLOAD	0xc0ffee01u
 #define GSP_PHYS_RMCALL_TIMEOUT_MS	2000u
 #define GSP_PHYS_RMCALL_POLL_MS		5u
@@ -7477,6 +7547,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
 		return "mode 26: golden9 9-entry promote on the V6 execution channel (plus the golden-channel promote)";
 	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY:
 		return "mode 27: golden9 9-entry promote on the V6 execution channel only (no golden-channel promote)";
+	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC:
+		return "mode 28: golden execution lane (golden channel with client GPFIFO/USERD, 9-entry promote on it, execution switched to it)";
 	default:
 		return "off (PROMOTE_CTX never sent)";
 	}
@@ -7851,6 +7923,14 @@ static void gsp_phys_rmcall_golden_3d_on_ladder(void __iomem *bar0)
 	if (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled() ||
 	    !boot_stage.rmcall_channel)
 		return;
+	/*
+	 * Mode 28: the golden execution channel already carries its TURING_A
+	 * object from the replay; the ladder's extra 3D object (a mode-25
+	 * GR-instance by-product) must not attach a second one to it.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC)
+		return;
 	ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
 			GSP_PHYS_RMCALL_CLASS_GR_3D,
 			GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER,
@@ -8068,6 +8148,7 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	struct gsp_phys_rmcall_channel_params ch;
 	struct gsp_phys_rmcall_mem_obj_shape shape;
 	u32 rpc = 0, status = 0;
+	u32 ch_parent = boot_stage.rmcall_device;
 	u64 golden_fb = 0;
 	bool pb_ok = true;
 	int ret;
@@ -8107,7 +8188,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) &&
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) &&
 	    gsp_phys_rmcall_engine_is_gr()) {
 		if (gsp_phys_rmcall_fwclient_mask_value() &
 		    GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) {
@@ -8211,7 +8294,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) &&
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) &&
 	    gsp_phys_rmcall_engine_is_gr()) {
 		ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9);
 		ch.internal_flags = cpu_to_le32(
@@ -8260,12 +8345,69 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC, 0x1000);
 		ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc_fb(
 			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_MTHD, 0x1000);
+		/*
+		 * Mode 28: the golden execution lane. Keep the captured shape
+		 * bit-for-bit (same captured flags 0x00200320, same FBMEM
+		 * instance/RAMFC/mthd slices) and only make the submission
+		 * path client-visible: the ladder GPFIFO ring at the proven VA
+		 * and the ladder sysmem USERD page (host GP_PUT writes land in
+		 * the hardware word; the captured USERD index 3 puts the
+		 * control block at userd+0x600, which
+		 * gsp_phys_rmcall_userd_ctl_off() applies). The channel stays
+		 * DEVICE-parented like the captured golden construct (a
+		 * golden9 promote on a TSG channel poisoned the next GR object
+		 * construct in modes 27/28 first run). Stage 3 swaps
+		 * boot_stage.rmcall_channel to this handle before the compute
+		 * object alloc.
+		 */
+		if (boot_stage.rmcall_gr_promote ==
+				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) {
+			u32 exec_shape =
+				gsp_phys_rmcall_golden_exec_shape_value();
+
+			if (exec_shape &
+			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_RING)
+				ch.gp_fifo_offset =
+					cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO);
+			if (exec_shape &
+			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_ENTRIES8)
+				ch.gp_fifo_entries = cpu_to_le32(
+					GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
+			if (exec_shape &
+			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SYSMEM) {
+				struct gsp_phys_rmcall_mem_desc ud =
+					gsp_phys_rmcall_mem_desc(
+						GSP_PHYS_RMCALL_BUF_USERD);
+
+				if (exec_shape &
+				    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SIZE200)
+					ud.size = cpu_to_le64(0x200);
+				ch.userd_mem = ud;
+			}
+			if (exec_shape &
+			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_HOBJ_VAS)
+				ch.h_object_buffer =
+					cpu_to_le32(boot_stage.rmcall_vas);
+		}
 		pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=FBMEM memdescs block=0x%llx (capture-matched apertures)\n",
 			GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
 			GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES, golden_fb);
+		if (boot_stage.rmcall_gr_promote ==
+				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC)
+			pr_info("vgsp-boot: rm-call: GOLDEN EXEC channel shape (mode 28): flags=0x%x (captured) shape=0x%x gpFifoOffset=0x%llx gpFifoEntries=%u hObjectBuffer=0x%x userd=%s size=0x%llx hParent=device hVASpace=0x%x; instance/ramfc/mthd stay FBMEM\n",
+				GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
+				gsp_phys_rmcall_golden_exec_shape_value(),
+				le64_to_cpu(ch.gp_fifo_offset),
+				le32_to_cpu(ch.gp_fifo_entries),
+				le32_to_cpu(ch.h_object_buffer),
+				(ch.userd_mem.address_space ==
+				 cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM)) ?
+					"sysmem" : "FBMEM",
+				le64_to_cpu(ch.userd_mem.size),
+				boot_stage.rmcall_vas);
 	}
 	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
-				    boot_stage.rmcall_device,
+				    ch_parent,
 				    GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
 				    GSP_PHYS_RMCALL_CLASS_CHANNEL,
 				    &ch, sizeof(ch), false, NULL,
@@ -8295,7 +8437,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	if ((boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6) &&
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) &&
 	    gsp_phys_rmcall_engine_is_gr()) {
 		ret = gsp_phys_rmcall_golden9_promote(bar0);
 		if (ret)
@@ -8303,6 +8447,13 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	}
 
 	/* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC &&
+	    (gsp_phys_rmcall_golden_exec_shape_value() &
+	     GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_SKIP_TURING_A)) {
+		pr_info("vgsp-boot: rm-call: mode 28: GOLDEN TURING_A 3D object skipped (shape bit5); the ladder's TURING_COMPUTE_A on the same channel provides the GR instance state\n");
+		return 0;
+	}
 	ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
 			GSP_PHYS_RMCALL_CLASS_GR_3D,
 			GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ,
@@ -9769,13 +9920,25 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 	 * GR-instance state to consume. Abort instead of sending it blind.
 	 */
 	if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
-	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) &&
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY ||
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) &&
 	    (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) {
 		pr_err("vgsp-boot: rm-call: mode %u requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
 		       mode);
 		return -EINVAL;
 	}
 	/*
+	 * Mode 28: the 9-entry promote already ran on the golden execution
+	 * channel inside the golden replay (it owns the committed context and
+	 * the forwarded launches); nothing is sent on the V6 channel at all.
+	 */
+	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) {
+		pr_info("vgsp-boot: rm-call: mode-28 golden execution lane: no PROMOTE_CTX on V6 (the golden channel 0x%x carries the committed context); continuing to stage 4 on the golden channel\n",
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL);
+		boot_stage.rmcall_gr_promote_step = 1;
+		return 0;
+	}
+	/*
 	 * Modes 8/9/10 must run on the V6 channel: without the
 	 * SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping
 	 * route that asserted GSP-RM in modes 1/2/4 (or rejects the
@@ -10734,7 +10897,64 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: UNRESTRICTED_PRIV_ACCESS_MAP omitted (allocation unavailable)\n");
 		}
 
-		if (n > 0) {
+		if (n > 0 && (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_INDIVIDUAL)) {
+			u32 k;
+			u32 n_single_ok = 0;
+
+			/*
+			 * Bisection: the batched non-MAIN control was refused
+			 * 0x1f as a whole (m2b/m2f); send each built entry as
+			 * its own one-entry control so one refused entry cannot
+			 * refuse the rest, and the accepted subset is visible
+			 * entry by entry.
+			 */
+			for (k = 0; k < n; k++) {
+				struct gsp_phys_rmcall_promote_entry e =
+					p->entry[k];
+
+				/*
+				 * Overwrite only slot 0: the entries k+1..n-1
+				 * must survive for their own iterations (a full
+				 * memset here destroyed the not-yet-sent ones).
+				 */
+				p->entry[0] = e;
+				p->entry_count = cpu_to_le32(1);
+				rpc_result = 0;
+				status = 0;
+				ret = gsp_phys_rmcall_control(bar0,
+						boot_stage.rmcall_subdevice,
+						GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+						p, sizeof(*p), 24, NULL, 0,
+						&rpc_result, &status,
+						"GPU_PROMOTE_CTX (FWCLIENT non-MAIN single)");
+				pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single[%u/%u]: bufferId=%u status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
+					k + 1, n, le16_to_cpu(e.buffer_id),
+					status, rpc_result,
+					boot_stage.rmcall_channel);
+				if (ret) {
+					/*
+					 * Bisection: keep going so the whole
+					 * acceptance matrix is visible in one
+					 * run; the ladder aborts only if no
+					 * entry was accepted at all.
+					 */
+					pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single[%u] bufferId=%u refused (status=0x%x rpc_result=0x%x, %s, ret=%d); continuing the bisection\n",
+						k, le16_to_cpu(e.buffer_id),
+						status, rpc_result,
+						gsp_phys_rmcall_status_str(status),
+						ret);
+					continue;
+				}
+				n_single_ok++;
+			}
+			if (!n_single_ok) {
+				pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single: no entry accepted; the GR context commit is incomplete and the ladder aborts before BIND with no retry\n");
+				kvfree(p);
+				return -EIO;
+			}
+			boot_stage.rmcall_gr_promote_step =
+				(mask & BIT(0)) ? 2 : 1;
+		} else if (n > 0) {
 			p->entry_count = cpu_to_le32(n);
 			rpc_result = 0;
 			status = 0;
@@ -10762,6 +10982,103 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 		} else if (!(mask & BIT(0))) {
 			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: MAIN omitted by mask\n");
 		}
+
+		/*
+		 * Bit 12: the golden9 VA-only global bookings, one control per
+		 * entry. Accepted entries add the GR engine's bundle/pagepool/
+		 * attribute/RTV reservations to the ladder channel's context;
+		 * a refused one is logged and skipped (bisection style).
+		 */
+		if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_GOLDEN9_GLOBALS) {
+			static const struct {
+				u16 id;
+				u64 va;
+				const char *name;
+			} gl[4] = {
+				{ 3, GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB,
+				  "BUNDLE_CB" },
+				{ 4, GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL,
+				  "PAGEPOOL" },
+				{ 5, GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB,
+				  "ATTRIBUTE_CB" },
+				{ 6, GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
+				  "RTV_CB_GLOBAL" },
+			};
+			u32 g;
+
+			for (g = 0; g < 4; g++) {
+				gsp_phys_rmcall_promote_entry_fill(p, 0,
+						gl[g].id, 0, gl[g].va, 0, 0,
+						0, 0);
+				p->entry_count = cpu_to_le32(1);
+				rpc_result = 0;
+				status = 0;
+				ret = gsp_phys_rmcall_control(bar0,
+						boot_stage.rmcall_subdevice,
+						GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+						p, sizeof(*p), 24, NULL, 0,
+						&rpc_result, &status,
+						"GPU_PROMOTE_CTX (golden9 global)");
+				pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9-global[%u/4] %s bufferId=%u va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
+					g + 1, gl[g].name, gl[g].id, gl[g].va,
+					status, rpc_result,
+					boot_stage.rmcall_channel);
+				if (ret)
+					pr_warn("vgsp-boot: rm-call: PROMOTE_CTX golden9-global[%u] %s refused (status=0x%x rpc_result=0x%x, %s, ret=%d); continuing the bisection\n",
+						g, gl[g].name, status,
+						rpc_result,
+						gsp_phys_rmcall_status_str(status),
+						ret);
+			}
+		}
+
+		/*
+		 * Bits 13/14: the FECS_EVENT halves. The combined form is
+		 * refused 0x1f on V6; a PA-only init entry and a VA-only
+		 * booking entry are each individually acceptable shapes on
+		 * this channel, and together they may carry the FECS state.
+		 */
+		if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY) {
+			if (boot_stage.rmcall_fecs_pa &&
+			    boot_stage.rmcall_fecs_size) {
+				gsp_phys_rmcall_promote_entry_fill(p, 0, 9,
+					boot_stage.rmcall_fecs_pa, 0,
+					boot_stage.rmcall_fecs_size, fattr,
+					1, 1);
+				p->entry_count = cpu_to_le32(1);
+				rpc_result = 0;
+				status = 0;
+				ret = gsp_phys_rmcall_control(bar0,
+						boot_stage.rmcall_subdevice,
+						GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+						p, sizeof(*p), 24, NULL, 0,
+						&rpc_result, &status,
+						"GPU_PROMOTE_CTX (FECS PA-only)");
+				pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS PA-only: bufferId=9 pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
+					boot_stage.rmcall_fecs_pa,
+					boot_stage.rmcall_fecs_size, fattr,
+					status, rpc_result,
+					boot_stage.rmcall_channel);
+			} else {
+				pr_warn("vgsp-boot: rm-call: PROMOTE_CTX FECS PA-only skipped: no FECS buffer prepared\n");
+			}
+		}
+		if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY) {
+			gsp_phys_rmcall_promote_entry_fill(p, 0, 9, 0,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, 0, 0, 0, 0);
+			p->entry_count = cpu_to_le32(1);
+			rpc_result = 0;
+			status = 0;
+			ret = gsp_phys_rmcall_control(bar0,
+					boot_stage.rmcall_subdevice,
+					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+					p, sizeof(*p), 24, NULL, 0,
+					&rpc_result, &status,
+					"GPU_PROMOTE_CTX (FECS VA-only)");
+			pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS VA-only: bufferId=9 va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
+				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, status,
+				rpc_result, boot_stage.rmcall_channel);
+		}
 	} else {
 		pr_err("vgsp-boot: rm-call: unknown gr_promote_param=%u reached the promote step; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
 		       mode);
@@ -10883,7 +11200,8 @@ static int gsp_phys_rmcall_srv_ctx_commit(void __iomem *bar0)
 	u64 main_pa, main_size;
 	int ret;
 
-	if (boot_stage.rmcall_srv_main_valid) {
+	if (boot_stage.rmcall_srv_main_valid &&
+	    !gsp_phys_srv_main_ignore_value()) {
 		main_pa = boot_stage.rmcall_srv_main_pa;
 		main_size = boot_stage.rmcall_srv_main_size;
 		/*
@@ -11367,11 +11685,13 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 	if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) {
 		ret = gsp_phys_rmcall_golden_replay(bar0);
 		if (ret) {
-			if (boot_stage.rmcall_gr_promote ==
-					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+			if ((boot_stage.rmcall_gr_promote ==
+					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
+			     boot_stage.rmcall_gr_promote ==
+					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) &&
 			    gsp_phys_rmcall_engine_is_gr()) {
-				pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-25 golden replay failed (%d); the golden construct is atomic and is not retried\n",
-				       ret);
+				pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-%u golden replay failed (%d); the golden construct is atomic and is not retried\n",
+				       boot_stage.rmcall_gr_promote, ret);
 				return ret;
 			}
 			pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n",
@@ -11413,6 +11733,29 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 		return ret;
 
 	/*
+	 * Mode 28: the golden execution lane. The golden replay allocated the
+	 * golden channel with a client GPFIFO/USERD and the accepted 9-entry
+	 * context commit; switch the ladder's execution channel to it BEFORE
+	 * the compute-object alloc so the 0xc5c0 object (the forwarded
+	 * launch's SET_OBJECT target), stage 4's BIND/SCHEDULE/token sequence,
+	 * the GR sem check and every forwarded publish all run on the golden
+	 * channel. The V6 channel stays allocated but is never bound or
+	 * scheduled, and its FWCLIENT promote is suppressed (mode 28 returns
+	 * early in gsp_phys_rmcall_gr_ctx_promote).
+	 */
+	if (gsp_phys_rmcall_engine_is_gr() &&
+	    boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) {
+		pr_info("vgsp-boot: rm-call: mode 28: execution channel switched from V6 0x%x to golden execution channel 0x%x (client ring gpFifoVa=0x%llx userd=sysmem, device-parented); the V6 channel stays unscheduled\n",
+			boot_stage.rmcall_channel,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+			GSP_PHYS_RMCALL_VA_GPFIFO);
+		boot_stage.rmcall_channel =
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL;
+		boot_stage.rmcall_channel_server_mem = false;
+	}
+
+	/*
 	 * H1: bind the compute engine object to the just-allocated GR channel
 	 * and commit the engine context before stage 3 returns. The TURING_
 	 * COMPUTE_A alloc rejection aborts the ladder (the GR method block's
@@ -11636,6 +11979,24 @@ struct gsp_phys_rmcall_ce_submit {
 };
 
 /*
+ * The hardware USERD control block (Nvc46fControl GP_PUT/GP_GET) sits at
+ * userd base + USERD_INDEX_VALUE * 0x200. The exec-shape channels (V6 and
+ * every ladder variant) use index 0, so their GP_PUT/GP_GET live at offset
+ * 0x8c/0x88 of the chunk. Mode 28 keeps the captured golden channel flags
+ * (USERD_INDEX_VALUE=3 within a fixed USERD page), so the hardware control
+ * block sits at userd+0x600 and every host USERD touch must use the same
+ * offset or host and hardware would poll different words.
+ */
+static u32 gsp_phys_rmcall_userd_ctl_off(void)
+{
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC &&
+	    gsp_phys_rmcall_engine_is_gr())
+		return GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED;
+	return 0;
+}
+
+/*
  * Shared publish tail of the ladder submissions: read USERD GP_PUT/GP_GET,
  * encode and write the GPFIFO entry for the module pushbuffer VA, advance
  * GP_PUT, snapshot the C3 fault epoch and ring the PF doorbell. Extracted
@@ -11669,8 +12030,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va,
 	if (!ring_va || !userd)
 		return -EIO;
 
-	gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
-	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
+	gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
+	gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
 	if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) {
 		pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n",
 		       gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
@@ -11705,7 +12066,7 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va,
 	 * the host write is therefore what the engine polls (cycle 21).
 	 */
 	put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES,
-			   (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
+			   (u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
 	/*
 	 * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) |
 	 * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated
@@ -11879,9 +12240,9 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
 		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
 			break;
 		if (time_after_eq(jiffies, deadline)) {
-			gpget = get_unaligned_le32((u8 *)userd +
+			gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() +
 						   GSP_PHYS_RMCALL_USERD_GPGET_OFF);
-			gpput = get_unaligned_le32((u8 *)userd +
+			gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() +
 						   GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
 			pr_err("vgsp-boot: rm-call: REAL GR SEM FAILED: no release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u engine=%s); the channel did not execute the GR block\n",
 			       GSP_PHYS_RMCALL_TIMEOUT_MS, poll,
@@ -11893,10 +12254,10 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
 		msleep(GSP_PHYS_RMCALL_POLL_MS);
 	}
 
-	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
+	gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
 	pr_info("vgsp-boot: rm-call: REAL GR SEM OK sem=0x%08x want=0x%08x GPGet=%u GPPut=%u words=%u token=0x%08x\n",
 		poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget,
-		get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
+		get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
 		sub.words, boot_stage.rmcall_token);
 	/*
 	 * Latch the GR execution gate: this is the only place a real GR
@@ -14059,9 +14420,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
 			break;
 		if (time_after_eq(jiffies, deadline)) {
-			gpget = get_unaligned_le32((u8 *)userd +
+			gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() +
 						   GSP_PHYS_RMCALL_USERD_GPGET_OFF);
-			gpput = get_unaligned_le32((u8 *)userd +
+			gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() +
 						   GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
 			pr_err("vgsp-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n",
 			       GSP_PHYS_RMCALL_TIMEOUT_MS, poll,
@@ -14087,10 +14448,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 		return -EIO;
 	}
 
-	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
+	gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
 	pr_info("vgsp-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n",
 		poll, sub.src_dma, sub.dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget,
-		get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
+		get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
 		sub.words, boot_stage.rmcall_token);
 	return 0;
 }