vgsp: add golden-replay mode 25 (captured client construct + atomic 9-entry promote)

Ladder mode 25 reproduces the real 570.144 host client's golden-image
construct on the golden replay handles: captured channel shape (flags
0x00200320, hObjectBuffer 0, gpFifoOffset 0, gpFifoEntries 0x20,
engineType 1, host memdesc substitutions), a single atomic 9-entry
GPU_PROMOTE_CTX on the golden channel (MAIN id0 and PATCH id2 combined,
ids 3/4/5/6 VA-only globals, ids 9/10/11 FECS/PRIV/UNR combined), then
the TURING_A 3D object. The golden VAS/channel/3D object stay alive and
the existing V6 FWCLIENT promote follows on the golden-initialized
GR-instance state; a failed golden step aborts with no retry and no
follow-up promote. The captured VAS COPY_SERVER_RESERVED_PDES control is
implemented gated (fwclient mask bit 8, zero-level no-op form) because
its level PAs are server-internal. Mode clamp/name/help extended to 25;
modes 1-24 unchanged.

3 files changed, 344 insertions, 15 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 0c2e75a..a25a0f0 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -330,6 +330,14 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES	0x801707u
 #define GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO		0x20801109u
 #define GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX		0x2080012bu
+/*
+ * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106), the golden
+ * client's VAS construct control. Mode 25 sends it only in the gated
+ * zero-level no-op-range form (see
+ * GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES); the captured three level PAs
+ * are server-RM-internal and have no safe client source.
+ */
+#define GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES	0x90f10106u
 #define GSP_PHYS_RMCALL_CTRL_GPU_EVICT_CTX		0x2080012cu
 /*
  * NV2080_CTRL_CMD_GPU_INITIALIZE_CTX (0x2080012d, ctrl2080gpu.h:1047-1063):
@@ -514,8 +522,33 @@ bool gsp_phys_boot_requested(void)
  * entryCount shrinks to the number of real entries.
  */
 #define GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT		24u
+/*
+ * Mode 25 (2026-10-05, opt-in): the real golden-client construct captured
+ * from the 570.144 host client (REPORT-GOLDEN-CAPTURE-DECODED), replayed on
+ * the golden handles in gsp_phys_rmcall_golden_replay: the captured VAS
+ * COPY_SERVER_RESERVED_PDES control (0x90f10106, gated by the fwclient mask
+ * bit 8 because the level PAs are server-internal), the captured channel
+ * shape (flags 0x00200320, hObjectBuffer 0, gpFifoOffset 0, gpFifoEntries
+ * 0x20, engineType 1, host-buffer memdescs) and the single atomic 9-entry
+ * GPU_PROMOTE_CTX on the golden channel (MAIN id0 combined, PATCH id2
+ * combined, ids 3/4/5/6 VA-only globals, ids 9/10/11 FECS/PRIV/UNR combined;
+ * sysmem physAttr 0x6 is the documented deviation). The golden VAS/channel/3D
+ * object stay alive and the ladder's own V6 FWCLIENT promote then follows on
+ * the golden-initialized GR-instance state. Requires gsp_phys_rmcall_golden=1
+ * plus gsp_phys_rmcall_engine=1; a failed golden step aborts the ladder with
+ * no retry and no follow-up promote.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9		25u
+/*
+ * Existing fwclient mask bit 8: mode-25 gate for the captured VAS
+ * COPY_SERVER_RESERVED_PDES control. The captured control copies three
+ * server-RM-internal page-table levels the ladder cannot source safely, so
+ * mode 25 sends only the zero-level no-op-range form and only when this bit
+ * is set; the default mask 0x3f leaves it off (skipped, logged).
+ */
+#define GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES	(1u << 8)
 #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
-	(GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)
+	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)
 
 /*
  * Work-submit token notifier slot. The default index is
@@ -793,6 +826,18 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_CTXBUFFER_ALLOC (1u << 31)
 
 /*
+ * Golden-image channel flag word captured from the real 570.144 host client
+ * (0x00200320): PRIVILEGED_CHANNEL (bit 5), USERD_INDEX_VALUE=3 (bits 10:8,
+ * the client's chId % 8) and USERD_INDEX_PAGE_FIXED (bit 21). SKIP_SCRUBBER
+ * (bit 23) and SKIP_CTXBUFFER_ALLOC (bit 31) are deliberately clear: the real
+ * golden channel is client-managed through the registry, not through the
+ * channel flags.
+ */
+#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9 \
+	(u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (3u << 8) | \
+	      (1u << 21))
+
+/*
  * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as
  * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc
  * RPC never replied, every later RPC timed out, and the fail-safe held the
@@ -6173,6 +6218,31 @@ struct gsp_phys_rmcall_promote_params {
 };
 
 /*
+ * NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS (184 bytes): hSubDevice,
+ * subDeviceId, pageSize, virtAddrLo, virtAddrHi, numLevelsToCopy and six level
+ * entries {physAddr, size, aperture, pageShift}. The ladder has no safe source
+ * for the server-RM level PAs the real client copied, so the gated mode-25
+ * path sends only the zero-level no-op-range form.
+ */
+struct gsp_phys_rmcall_vas_copy_pdes_level {
+	__le64 phys_addr;
+	__le64 size;
+	__le32 aperture;
+	__le32 page_shift;
+};
+
+struct gsp_phys_rmcall_vas_copy_pdes_params {
+	__le32 h_sub_device;
+	__le32 sub_device_id;
+	__le64 page_size;
+	__le64 virt_addr_lo;
+	__le64 virt_addr_hi;
+	__le32 num_levels_to_copy;
+	__le32 pad;
+	struct gsp_phys_rmcall_vas_copy_pdes_level levels[6];
+};
+
+/*
  * NV2080_CTRL_GPU_EVICT_CTX_PARAMS (20 bytes,
  * ctrl/ctrl2080/ctrl2080gpu.h:1000-1006): engineType/hClient/ChID/
  * hChanClient/hObject. Evicts the channel's virtual engine context; on the
@@ -7241,6 +7311,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
 		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote";
 	case GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT:
 		return "reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) on V6";
+	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9:
+		return "real golden-client construct on the golden handles (captured channel 0x00200320 + single atomic 9-entry PROMOTE_CTX) then the V6 FWCLIENT follow-on promote";
 	default:
 		return "off (PROMOTE_CTX never sent)";
 	}
@@ -7248,7 +7320,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
 
 /*
  * H1 GR context-promote bisection snapshot. gsp_phys_rmcall_gr_promote is a
- * load-time module parameter (0..13, see gsp_phys_gpu.c); stage 3 snapshots it
+ * load-time module parameter (0..GSP_PHYS_RMCALL_GR_PROMOTE_MAX, see
+ * gsp_phys_gpu.c); stage 3 snapshots it
  * next to the engine so every later step of this staging uses one shape. The
  * module parameter's set callback already clamps out-of-range values with a
  * warning; the check here is a second line of defense against a future
@@ -7424,7 +7497,9 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)) {
+			GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)) {
 		vars = &gsp_phys_rmcall_channel_variant_skipctx;
 		attempts = 1;
 	}
@@ -7668,6 +7743,48 @@ static void gsp_phys_rmcall_golden_fail(const char *step, u32 handle,
 	       step, handle, h_class, status, rpc, ret);
 }
 
+/*
+ * Mode-25 VAS construct control (0x90f10106), gated by fwclient mask bit 8.
+ * The captured real-client control copied three server-RM-internal page-table
+ * levels; the ladder has no safe source for those PAs, so the only form this
+ * path can send is the zero-level no-op range (numLevelsToCopy=0) and it does
+ * so only when the gate is set. With the default mask the control is skipped
+ * and the deviation is logged; this sub-step still needs a live bisection
+ * (report open item 3).
+ */
+static int gsp_phys_rmcall_golden9_vas_control(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_vas_copy_pdes_params *p;
+	u32 rpc_result = 0, status = 0;
+	int ret;
+
+	if (!(gsp_phys_rmcall_fwclient_mask_value() &
+	      GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES)) {
+		pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped: the captured three level PAs are server-RM-internal and the ladder has no safe source (uncertain, report open item 3); set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the zero-level no-op-range form for bisection\n",
+			GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES);
+		return 0;
+	}
+
+	p = kvzalloc(sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+	p->page_size = cpu_to_le64(1ULL << 29);	/* GMMU_PD1_VADDR_BIT_LO */
+	p->virt_addr_hi = cpu_to_le64(0x11fffffffULL);
+	p->num_levels_to_copy = cpu_to_le32(0);
+	/* The captured RPC targets the VAS itself (hObject=0xbaba0042). */
+	ret = gsp_phys_rmcall_control(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
+			GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES,
+			p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
+			"VASPACE_COPY_SERVER_RESERVED_PDES (no-op range, gated)");
+	pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES: no-op range numLevels=0 pageSize=0x20000000 virtAddrHi=0x11fffffff status=0x%x rpc_result=0x%x ret=%d\n",
+		status, rpc_result, ret);
+	kvfree(p);
+	return ret;
+}
+
+static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0);
+
 static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
 				      const struct gsp_phys_rmcall_mem_obj_shape *shape,
 				      const char *step)
@@ -7718,6 +7835,22 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 		return ret;
 	}
 
+	/*
+	 * Mode 25: the captured VAS construct control follows the VAS alloc.
+	 * It is gated off by default and aborts the replay only when the
+	 * operator enabled the gate and the server rejected it.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+	    gsp_phys_rmcall_engine_is_gr()) {
+		ret = gsp_phys_rmcall_golden9_vas_control(bar0);
+		if (ret) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9 VAS control failed (%d); the golden replay aborts with no retry\n",
+			       ret);
+			return ret;
+		}
+	}
+
 	/* 2. GPFIFO pushbuffer, physical (kernel_graphics.c:2202-2222). */
 	memset(&shape, 0, sizeof(shape));
 	shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS;
@@ -7774,6 +7907,29 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	ch.h_userd_memory[0] =
 		cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD);
 	ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+	/*
+	 * Mode 25: the captured golden channel shape. The real client's
+	 * instance/USERD/RAMFC/mthd blocks were client-local memmgr
+	 * allocations (FBMEM in the capture); the ladder substitutes the
+	 * registered host sysmem chunks and documents the deviation.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+	    gsp_phys_rmcall_engine_is_gr()) {
+		ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9);
+		ch.h_object_buffer = cpu_to_le32(0);
+		ch.instance_mem = gsp_phys_rmcall_mem_desc(
+			GSP_PHYS_RMCALL_BUF_INSTANCE);
+		ch.userd_mem = gsp_phys_rmcall_mem_desc(
+			GSP_PHYS_RMCALL_BUF_USERD);
+		ch.ramfc_mem = gsp_phys_rmcall_mem_desc(
+			GSP_PHYS_RMCALL_BUF_RAMFC);
+		ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc(
+			GSP_PHYS_RMCALL_BUF_MTHDBUF);
+		pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=host sysmem memdescs (capture used client-local FBMEM)\n",
+			GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
+			GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES);
+	}
 	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
 				    boot_stage.rmcall_device,
 				    GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
@@ -7793,6 +7949,23 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 		return ret;
 	}
 
+	/*
+	 * Mode 25: the single atomic 9-entry GPU_PROMOTE_CTX on the golden
+	 * channel, in the captured order/forms. It runs after the channel
+	 * alloc and before the 3D object, matching the wire order of the real
+	 * client (its GR-object constructor promotes before it allocates the
+	 * object). A failure aborts the whole golden replay with no retry and
+	 * no follow-up promote; the golden VAS/channel/3D objects stay alive
+	 * for the ladder's own V6 FWCLIENT follow-on promote.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+	    gsp_phys_rmcall_engine_is_gr()) {
+		ret = gsp_phys_rmcall_golden9_promote(bar0);
+		if (ret)
+			return ret;
+	}
+
 	/* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */
 	ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
 			GSP_PHYS_RMCALL_CLASS_GR_3D,
@@ -8543,6 +8716,123 @@ static void gsp_phys_rmcall_gr_fwclient_small_prepare(void)
 }
 
 /*
+ * Mode-25 golden 9-entry promote (0x2080012b): ONE atomic control on the
+ * golden channel with the captured entry order/forms - MAIN id0 combined,
+ * PATCH id2 combined, ids 3/4/5/6 VA-only globals (BUNDLE_CB, PAGEPOOL,
+ * ATTRIBUTE_CB, RTV_CB_GLOBAL) and ids 9/10/11 (FECS_EVENT, PRIV_ACCESS_MAP,
+ * UNRESTRICTED_PRIV_ACCESS_MAP) combined. The ladder's prepared sysmem
+ * context buffers back the combined entries (physAttr 0x6, the documented
+ * deviation from the capture's VIDMEM 0x4); the VA-only globals keep
+ * booking-only VAs above the context window. All nine buffers must be
+ * present; a short set or a rejected control aborts the whole replay with no
+ * retry and never a partial accept followed by another promote.
+ */
+static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_promote_params *p;
+	u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
+	u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
+	u32 rpc_result = 0, status = 0;
+	unsigned int i;
+	int ret;
+
+	if (!boot_stage.rmcall_ctx_prepared ||
+	    !boot_stage.rmcall_ctx_pa || !boot_stage.rmcall_ctx_size) {
+		pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: no prepared MAIN context buffer (prepared=%u pa=0x%llx size=0x%x)\n",
+		       boot_stage.rmcall_ctx_prepared,
+		       boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size);
+		return -EINVAL;
+	}
+	ret = gsp_phys_rmcall_gr_patch_prepare(bar0);
+	if (ret)
+		return ret;
+	gsp_phys_rmcall_gr_fwclient_small_prepare();
+	if (!boot_stage.rmcall_patch_pa || !boot_stage.rmcall_fecs_pa ||
+	    !boot_stage.rmcall_priv_pa ||
+	    !boot_stage.rmcall_unrestricted_pa) {
+		pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: patch=0x%llx fecs=0x%llx priv=0x%llx unr=0x%llx (all nine buffers must be present); no promote is sent and the ladder aborts with no retry\n",
+		       boot_stage.rmcall_patch_pa, boot_stage.rmcall_fecs_pa,
+		       boot_stage.rmcall_priv_pa,
+		       boot_stage.rmcall_unrestricted_pa);
+		return -ENOMEM;
+	}
+
+	p = kvzalloc(sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+
+	p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+	p->h_client = cpu_to_le32(0);
+	p->ch_id = cpu_to_le32(0);
+	p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
+	p->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL);
+	p->entry_count = cpu_to_le32(9);
+
+	gsp_phys_rmcall_promote_entry_fill(p, 0,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
+		boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
+		boot_stage.rmcall_ctx_size, attr, 1, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 1,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
+		boot_stage.rmcall_patch_pa, GSP_PHYS_RMCALL_VA_GR_PATCH,
+		boot_stage.rmcall_patch_size, attr, 1, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 2,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_BUNDLE_CB,
+		0, bk + 0x2000ULL, 0, 0, 0, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 3,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PAGEPOOL,
+		0, bk + 0x3000ULL, 0, 0, 0, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 4,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_ATTRIBUTE_CB,
+		0, bk + 0x4000ULL, 0, 0, 0, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 5,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_RTV_CB_GLOBAL,
+		0, bk + 0x5000ULL, 0, 0, 0, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 6,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
+		boot_stage.rmcall_fecs_pa, bk + 0x6000ULL, 0x1000u, attr, 1, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 7,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
+		boot_stage.rmcall_priv_pa, bk + 0x7000ULL, 0x1000u, attr, 1, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 8,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
+		boot_stage.rmcall_unrestricted_pa, bk + 0x8000ULL,
+		0x1000u, attr, 1, 0);
+
+	for (i = 0; i < 9; i++)
+		pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9 entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s\n",
+			i, le16_to_cpu(p->entry[i].buffer_id),
+			le64_to_cpu(p->entry[i].gpu_phys_addr),
+			le64_to_cpu(p->entry[i].gpu_virt_addr),
+			le64_to_cpu(p->entry[i].size),
+			le32_to_cpu(p->entry[i].phys_attr),
+			p->entry[i].b_initialize, p->entry[i].b_nonmapped,
+			(i == 2 || i == 3 || i == 4 || i == 5) ?
+				" - VA-only booking, no PA" : "");
+
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
+			GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+			p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
+			"GPU_PROMOTE_CTX (golden9, 9 entries, atomic)");
+	pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9: entryCount=9 status=0x%x rpc_result=0x%x (hObject=golden channel 0x%x hChanClient=0x%x engineType=0x%x MAIN pa=0x%llx va=0x%llx size=0x%x; gr_promote_param=%u)\n",
+		status, rpc_result, GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+		boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0,
+		boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
+		boot_stage.rmcall_ctx_size,
+		GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9);
+	if (ret) {
+		pr_err("vgsp-boot: rm-call: PROMOTE_CTX golden9 (9 entries) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the golden context is not committed and the ladder aborts with no retry and no follow-up promote\n",
+		       status, rpc_result,
+		       gsp_phys_rmcall_status_str(status), ret);
+		kvfree(p);
+		return ret;
+	}
+	boot_stage.rmcall_gr_promote_step = 1;
+	kvfree(p);
+	return 0;
+}
+
+/*
  * Fill one MAIN promote entry with the prepared PA/VA and the snapshot's
  * entry size. initialize/nonmapped carry the bisection shape: modes 1/3/5
  * entry 0 and mode 2 step 1 use a full field set; mode 2 step 2 repeats the
@@ -8676,6 +8966,11 @@ static int gsp_phys_rmcall_alloc_gr_hvirt(void __iomem *bar0)
  *     entry in the combined kgrobjPromoteContext form where the ladder has
  *     a VA; PM and PRIV_ACCESS_MAP stay init-only and unsourceable buffers
  *     are omitted, all logged per entry.
+ *   - mode 25: the same follow-on FWCLIENT promote on the ladder's V6
+ *     channel, after gsp_phys_rmcall_golden_replay replayed the real golden
+ *     construct (captured VAS control, captured channel shape and the single
+ *     atomic 9-entry PROMOTE_CTX on the golden channel) so the server-side
+ *     GR-instance state exists before this control runs.
  * Mode 0 never reaches this function (stage 3 stops before it); the early
  * return is a belt-and-braces no-op. Success means the server installed the
  * context memdesc, added the VA to the engine-context vaList, committed the
@@ -8708,6 +9003,17 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OFF)
 		return -EOPNOTSUPP;
 	/*
+	 * Mode 25's golden construct lives in the golden replay
+	 * (gsp_phys_rmcall_golden_replay); without both gates the replay never
+	 * ran, so the follow-on FWCLIENT promote would have no golden-initialized
+	 * GR-instance state to consume. Abort instead of sending it blind.
+	 */
+	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+	    (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) {
+		pr_err("vgsp-boot: rm-call: mode 25 requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n");
+		return -EINVAL;
+	}
+	/*
 	 * Modes 8/9/10 must run on the V6 channel: without the
 	 * SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping
 	 * route that asserted GSP-RM in modes 1/2/4 (or rejects the
@@ -8726,7 +9032,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
-	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) &&
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) &&
 	    boot_stage.rmcall_channel_variant !=
 			gsp_phys_rmcall_channel_variant_skipctx.id) {
 		pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
@@ -9406,7 +9713,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 			return ret;
 		}
 		boot_stage.rmcall_gr_promote_step = 1;
-	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) {
+	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) {
 		/*
 		 * Mode 24: the reference FWCLIENT non-3D set as ONE control.
 		 * Order and bufferIds follow kgrobjGetPromoteIds_FWCLIENT
@@ -9423,7 +9731,9 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 		 * stock shape is init-only too; both deviations are logged
 		 * per entry. Auxiliary buffers that cannot be sourced are
 		 * omitted and logged; entryCount is the number of real
-		 * entries.
+		 * entries. Mode 25 shares this follow-on promote on the
+		 * ladder's V6 channel after its golden construct; the
+		 * existing mask selects its entries too (bits 0-7).
 		 */
 		u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
 		u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
@@ -10236,15 +10546,25 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 	 * replay the stock client's post-load golden channel allocations after
 	 * the ladder's VAS/TSG exist and before the ladder's own channel/GR
 	 * object, so the server-side GR-instance state the GR engine-context
-	 * commit needs is initialized first. The first failure aborts the whole
-	 * ladder with no retry; with the parameter 0 this block is skipped
-	 * entirely (no RPC, no log).
+	 * commit needs is initialized first. With mode 25 the replay carries
+	 * the single atomic 9-entry promote and is no longer best-effort: a
+	 * failure aborts the whole ladder with no retry and no follow-up
+	 * promote. For every other mode the replay keeps its historical
+	 * best-effort continuation.
 	 */
 	if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) {
 		ret = gsp_phys_rmcall_golden_replay(bar0);
-		if (ret)
+		if (ret) {
+			if (boot_stage.rmcall_gr_promote ==
+					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+			    gsp_phys_rmcall_engine_is_gr()) {
+				pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-25 golden replay failed (%d); the golden construct is atomic and is not retried\n",
+				       ret);
+				return ret;
+			}
 			pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n",
 				ret);
+		}
 	}
 
 	/*
@@ -14500,6 +14820,11 @@ static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry_count) == 40
 	      "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.entryCount must be at offset 40");
 static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry) == 48,
 	      "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.promoteEntry must be at offset 48 (8-byte aligned)");
+static_assert(sizeof(struct gsp_phys_rmcall_vas_copy_pdes_params) == 184,
+	      "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS must be 184 bytes");
+static_assert(offsetof(struct gsp_phys_rmcall_vas_copy_pdes_params,
+		       num_levels_to_copy) == 32,
+	      "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS.numLevelsToCopy must be at offset 32");
 static_assert(sizeof(struct gsp_phys_rmcall_evict_ctx_params) == 20,
 	      "NV2080_CTRL_GPU_EVICT_CTX_PARAMS must be 20 bytes");
 static_assert(sizeof(struct gsp_phys_rmcall_ctx_buffer_info) == 80,
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 5e6b264..21313a2 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -511,8 +511,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val,
 		return ret;
 
 	if (gsp_phys_rmcall_gr_promote_param < 0 ||
-	    gsp_phys_rmcall_gr_promote_param > 24) {
-		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..24; clamping to 0 (no PROMOTE_CTX)\n",
+	    gsp_phys_rmcall_gr_promote_param > 25) {
+		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..25; clamping to 0 (no PROMOTE_CTX)\n",
 			gsp_phys_rmcall_gr_promote_param);
 		gsp_phys_rmcall_gr_promote_param = 0;
 	}
@@ -531,7 +531,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote,
 		&gsp_phys_rmcall_gr_promote_ops,
 		&gsp_phys_rmcall_gr_promote_param, 0644);
 MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote,
-		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); values outside 0..24 clamp to 0 with a warning");
+		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; values outside 0..25 clamp to 0 with a warning");
 
 int gsp_phys_rmcall_gr_promote_value(void)
 {
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 9e809e3..a38067a 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -250,8 +250,12 @@ int gsp_phys_rmcall_engine_value(void);
  *       PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) as one control on the
  *       V6 KMD-managed channel: combined PA+VA entries where the ladder has
  *       a VA, PM/PRIV_ACCESS_MAP init-only, unsourceable buffers omitted
- *       and logged.
- * The parameter's set callback clamps any value outside 0..24 to 0 with a
+ *       and logged;
+ *   25 = the real golden-client construct on the golden replay handles
+ *       (captured channel shape + single atomic 9-entry PROMOTE_CTX on the
+ *       golden channel) followed by the V6 FWCLIENT promote; requires
+ *       gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1.
+ * The parameter's set callback clamps any value outside 0..25 to 0 with a
  * warning and warns on every nonzero enable (a failed promote can assert
  * GSP-RM and wedge the device; recovery needs a reboot with human approval).
  * The booter snapshots the value when the rm-call ladder's stage 3 starts