From fc8d7923c83c6babd0ee8c23565ac18b709fa5a5 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 17:16:18 -0400
Subject: vgsp: add golden-replay mode 25 (captured client construct + atomic
 9-entry promote)

Ladder mode 25 reproduces the real 570.144 host client's golden-image
construct on the golden replay handles: captured channel shape (flags
0x00200320, hObjectBuffer 0, gpFifoOffset 0, gpFifoEntries 0x20,
engineType 1, host memdesc substitutions), a single atomic 9-entry
GPU_PROMOTE_CTX on the golden channel (MAIN id0 and PATCH id2 combined,
ids 3/4/5/6 VA-only globals, ids 9/10/11 FECS/PRIV/UNR combined), then
the TURING_A 3D object. The golden VAS/channel/3D object stay alive and
the existing V6 FWCLIENT promote follows on the golden-initialized
GR-instance state; a failed golden step aborts with no retry and no
follow-up promote. The captured VAS COPY_SERVER_RESERVED_PDES control is
implemented gated (fwclient mask bit 8, zero-level no-op form) because
its level PAs are server-internal. Mode clamp/name/help extended to 25;
modes 1-24 unchanged.
---
 kernel/gsp_phys_booter.c | 345 +++++++++++++++++++++++++++++++++++++++++++++--
 kernel/gsp_phys_gpu.c    |   6 +-
 kernel/gsp_phys_gpu.h    |   8 +-
 3 files changed, 344 insertions(+), 15 deletions(-)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 0c2e75a..a25a0f0 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -330,6 +330,14 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES	0x801707u
 #define GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO		0x20801109u
 #define GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX		0x2080012bu
+/*
+ * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106), the golden
+ * client's VAS construct control. Mode 25 sends it only in the gated
+ * zero-level no-op-range form (see
+ * GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES); the captured three level PAs
+ * are server-RM-internal and have no safe client source.
+ */
+#define GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES	0x90f10106u
 #define GSP_PHYS_RMCALL_CTRL_GPU_EVICT_CTX		0x2080012cu
 /*
  * NV2080_CTRL_CMD_GPU_INITIALIZE_CTX (0x2080012d, ctrl2080gpu.h:1047-1063):
@@ -514,8 +522,33 @@ bool gsp_phys_boot_requested(void)
  * entryCount shrinks to the number of real entries.
  */
 #define GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT		24u
+/*
+ * Mode 25 (2026-10-05, opt-in): the real golden-client construct captured
+ * from the 570.144 host client (REPORT-GOLDEN-CAPTURE-DECODED), replayed on
+ * the golden handles in gsp_phys_rmcall_golden_replay: the captured VAS
+ * COPY_SERVER_RESERVED_PDES control (0x90f10106, gated by the fwclient mask
+ * bit 8 because the level PAs are server-internal), the captured channel
+ * shape (flags 0x00200320, hObjectBuffer 0, gpFifoOffset 0, gpFifoEntries
+ * 0x20, engineType 1, host-buffer memdescs) and the single atomic 9-entry
+ * GPU_PROMOTE_CTX on the golden channel (MAIN id0 combined, PATCH id2
+ * combined, ids 3/4/5/6 VA-only globals, ids 9/10/11 FECS/PRIV/UNR combined;
+ * sysmem physAttr 0x6 is the documented deviation). The golden VAS/channel/3D
+ * object stay alive and the ladder's own V6 FWCLIENT promote then follows on
+ * the golden-initialized GR-instance state. Requires gsp_phys_rmcall_golden=1
+ * plus gsp_phys_rmcall_engine=1; a failed golden step aborts the ladder with
+ * no retry and no follow-up promote.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9		25u
+/*
+ * Existing fwclient mask bit 8: mode-25 gate for the captured VAS
+ * COPY_SERVER_RESERVED_PDES control. The captured control copies three
+ * server-RM-internal page-table levels the ladder cannot source safely, so
+ * mode 25 sends only the zero-level no-op-range form and only when this bit
+ * is set; the default mask 0x3f leaves it off (skipped, logged).
+ */
+#define GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES	(1u << 8)
 #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
-	(GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)
+	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)
 
 /*
  * Work-submit token notifier slot. The default index is
@@ -792,6 +825,18 @@ bool gsp_phys_boot_requested(void)
  */
 #define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_CTXBUFFER_ALLOC (1u << 31)
 
+/*
+ * Golden-image channel flag word captured from the real 570.144 host client
+ * (0x00200320): PRIVILEGED_CHANNEL (bit 5), USERD_INDEX_VALUE=3 (bits 10:8,
+ * the client's chId % 8) and USERD_INDEX_PAGE_FIXED (bit 21). SKIP_SCRUBBER
+ * (bit 23) and SKIP_CTXBUFFER_ALLOC (bit 31) are deliberately clear: the real
+ * golden channel is client-managed through the registry, not through the
+ * channel flags.
+ */
+#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9 \
+	(u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (3u << 8) | \
+	      (1u << 21))
+
 /*
  * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as
  * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc
@@ -6172,6 +6217,31 @@ struct gsp_phys_rmcall_promote_params {
 	struct gsp_phys_rmcall_promote_entry entry[16];
 };
 
+/*
+ * NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS (184 bytes): hSubDevice,
+ * subDeviceId, pageSize, virtAddrLo, virtAddrHi, numLevelsToCopy and six level
+ * entries {physAddr, size, aperture, pageShift}. The ladder has no safe source
+ * for the server-RM level PAs the real client copied, so the gated mode-25
+ * path sends only the zero-level no-op-range form.
+ */
+struct gsp_phys_rmcall_vas_copy_pdes_level {
+	__le64 phys_addr;
+	__le64 size;
+	__le32 aperture;
+	__le32 page_shift;
+};
+
+struct gsp_phys_rmcall_vas_copy_pdes_params {
+	__le32 h_sub_device;
+	__le32 sub_device_id;
+	__le64 page_size;
+	__le64 virt_addr_lo;
+	__le64 virt_addr_hi;
+	__le32 num_levels_to_copy;
+	__le32 pad;
+	struct gsp_phys_rmcall_vas_copy_pdes_level levels[6];
+};
+
 /*
  * NV2080_CTRL_GPU_EVICT_CTX_PARAMS (20 bytes,
  * ctrl/ctrl2080/ctrl2080gpu.h:1000-1006): engineType/hClient/ChID/
@@ -7241,6 +7311,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
 		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote";
 	case GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT:
 		return "reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) on V6";
+	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9:
+		return "real golden-client construct on the golden handles (captured channel 0x00200320 + single atomic 9-entry PROMOTE_CTX) then the V6 FWCLIENT follow-on promote";
 	default:
 		return "off (PROMOTE_CTX never sent)";
 	}
@@ -7248,7 +7320,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
 
 /*
  * H1 GR context-promote bisection snapshot. gsp_phys_rmcall_gr_promote is a
- * load-time module parameter (0..13, see gsp_phys_gpu.c); stage 3 snapshots it
+ * load-time module parameter (0..GSP_PHYS_RMCALL_GR_PROMOTE_MAX, see
+ * gsp_phys_gpu.c); stage 3 snapshots it
  * next to the engine so every later step of this staging uses one shape. The
  * module parameter's set callback already clamps out-of-range values with a
  * warning; the check here is a second line of defense against a future
@@ -7424,7 +7497,9 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)) {
+			GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)) {
 		vars = &gsp_phys_rmcall_channel_variant_skipctx;
 		attempts = 1;
 	}
@@ -7668,6 +7743,48 @@ static void gsp_phys_rmcall_golden_fail(const char *step, u32 handle,
 	       step, handle, h_class, status, rpc, ret);
 }
 
+/*
+ * Mode-25 VAS construct control (0x90f10106), gated by fwclient mask bit 8.
+ * The captured real-client control copied three server-RM-internal page-table
+ * levels; the ladder has no safe source for those PAs, so the only form this
+ * path can send is the zero-level no-op range (numLevelsToCopy=0) and it does
+ * so only when the gate is set. With the default mask the control is skipped
+ * and the deviation is logged; this sub-step still needs a live bisection
+ * (report open item 3).
+ */
+static int gsp_phys_rmcall_golden9_vas_control(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_vas_copy_pdes_params *p;
+	u32 rpc_result = 0, status = 0;
+	int ret;
+
+	if (!(gsp_phys_rmcall_fwclient_mask_value() &
+	      GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES)) {
+		pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped: the captured three level PAs are server-RM-internal and the ladder has no safe source (uncertain, report open item 3); set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the zero-level no-op-range form for bisection\n",
+			GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES);
+		return 0;
+	}
+
+	p = kvzalloc(sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+	p->page_size = cpu_to_le64(1ULL << 29);	/* GMMU_PD1_VADDR_BIT_LO */
+	p->virt_addr_hi = cpu_to_le64(0x11fffffffULL);
+	p->num_levels_to_copy = cpu_to_le32(0);
+	/* The captured RPC targets the VAS itself (hObject=0xbaba0042). */
+	ret = gsp_phys_rmcall_control(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
+			GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES,
+			p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
+			"VASPACE_COPY_SERVER_RESERVED_PDES (no-op range, gated)");
+	pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES: no-op range numLevels=0 pageSize=0x20000000 virtAddrHi=0x11fffffff status=0x%x rpc_result=0x%x ret=%d\n",
+		status, rpc_result, ret);
+	kvfree(p);
+	return ret;
+}
+
+static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0);
+
 static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
 				      const struct gsp_phys_rmcall_mem_obj_shape *shape,
 				      const char *step)
@@ -7718,6 +7835,22 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 		return ret;
 	}
 
+	/*
+	 * Mode 25: the captured VAS construct control follows the VAS alloc.
+	 * It is gated off by default and aborts the replay only when the
+	 * operator enabled the gate and the server rejected it.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+	    gsp_phys_rmcall_engine_is_gr()) {
+		ret = gsp_phys_rmcall_golden9_vas_control(bar0);
+		if (ret) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9 VAS control failed (%d); the golden replay aborts with no retry\n",
+			       ret);
+			return ret;
+		}
+	}
+
 	/* 2. GPFIFO pushbuffer, physical (kernel_graphics.c:2202-2222). */
 	memset(&shape, 0, sizeof(shape));
 	shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS;
@@ -7774,6 +7907,29 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	ch.h_userd_memory[0] =
 		cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD);
 	ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+	/*
+	 * Mode 25: the captured golden channel shape. The real client's
+	 * instance/USERD/RAMFC/mthd blocks were client-local memmgr
+	 * allocations (FBMEM in the capture); the ladder substitutes the
+	 * registered host sysmem chunks and documents the deviation.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+	    gsp_phys_rmcall_engine_is_gr()) {
+		ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9);
+		ch.h_object_buffer = cpu_to_le32(0);
+		ch.instance_mem = gsp_phys_rmcall_mem_desc(
+			GSP_PHYS_RMCALL_BUF_INSTANCE);
+		ch.userd_mem = gsp_phys_rmcall_mem_desc(
+			GSP_PHYS_RMCALL_BUF_USERD);
+		ch.ramfc_mem = gsp_phys_rmcall_mem_desc(
+			GSP_PHYS_RMCALL_BUF_RAMFC);
+		ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc(
+			GSP_PHYS_RMCALL_BUF_MTHDBUF);
+		pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=host sysmem memdescs (capture used client-local FBMEM)\n",
+			GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
+			GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES);
+	}
 	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
 				    boot_stage.rmcall_device,
 				    GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
@@ -7793,6 +7949,23 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 		return ret;
 	}
 
+	/*
+	 * Mode 25: the single atomic 9-entry GPU_PROMOTE_CTX on the golden
+	 * channel, in the captured order/forms. It runs after the channel
+	 * alloc and before the 3D object, matching the wire order of the real
+	 * client (its GR-object constructor promotes before it allocates the
+	 * object). A failure aborts the whole golden replay with no retry and
+	 * no follow-up promote; the golden VAS/channel/3D objects stay alive
+	 * for the ladder's own V6 FWCLIENT follow-on promote.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+	    gsp_phys_rmcall_engine_is_gr()) {
+		ret = gsp_phys_rmcall_golden9_promote(bar0);
+		if (ret)
+			return ret;
+	}
+
 	/* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */
 	ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
 			GSP_PHYS_RMCALL_CLASS_GR_3D,
@@ -8542,6 +8715,123 @@ static void gsp_phys_rmcall_gr_fwclient_small_prepare(void)
 	}
 }
 
+/*
+ * Mode-25 golden 9-entry promote (0x2080012b): ONE atomic control on the
+ * golden channel with the captured entry order/forms - MAIN id0 combined,
+ * PATCH id2 combined, ids 3/4/5/6 VA-only globals (BUNDLE_CB, PAGEPOOL,
+ * ATTRIBUTE_CB, RTV_CB_GLOBAL) and ids 9/10/11 (FECS_EVENT, PRIV_ACCESS_MAP,
+ * UNRESTRICTED_PRIV_ACCESS_MAP) combined. The ladder's prepared sysmem
+ * context buffers back the combined entries (physAttr 0x6, the documented
+ * deviation from the capture's VIDMEM 0x4); the VA-only globals keep
+ * booking-only VAs above the context window. All nine buffers must be
+ * present; a short set or a rejected control aborts the whole replay with no
+ * retry and never a partial accept followed by another promote.
+ */
+static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_promote_params *p;
+	u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
+	u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
+	u32 rpc_result = 0, status = 0;
+	unsigned int i;
+	int ret;
+
+	if (!boot_stage.rmcall_ctx_prepared ||
+	    !boot_stage.rmcall_ctx_pa || !boot_stage.rmcall_ctx_size) {
+		pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: no prepared MAIN context buffer (prepared=%u pa=0x%llx size=0x%x)\n",
+		       boot_stage.rmcall_ctx_prepared,
+		       boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size);
+		return -EINVAL;
+	}
+	ret = gsp_phys_rmcall_gr_patch_prepare(bar0);
+	if (ret)
+		return ret;
+	gsp_phys_rmcall_gr_fwclient_small_prepare();
+	if (!boot_stage.rmcall_patch_pa || !boot_stage.rmcall_fecs_pa ||
+	    !boot_stage.rmcall_priv_pa ||
+	    !boot_stage.rmcall_unrestricted_pa) {
+		pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: patch=0x%llx fecs=0x%llx priv=0x%llx unr=0x%llx (all nine buffers must be present); no promote is sent and the ladder aborts with no retry\n",
+		       boot_stage.rmcall_patch_pa, boot_stage.rmcall_fecs_pa,
+		       boot_stage.rmcall_priv_pa,
+		       boot_stage.rmcall_unrestricted_pa);
+		return -ENOMEM;
+	}
+
+	p = kvzalloc(sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+
+	p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+	p->h_client = cpu_to_le32(0);
+	p->ch_id = cpu_to_le32(0);
+	p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
+	p->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL);
+	p->entry_count = cpu_to_le32(9);
+
+	gsp_phys_rmcall_promote_entry_fill(p, 0,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
+		boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
+		boot_stage.rmcall_ctx_size, attr, 1, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 1,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
+		boot_stage.rmcall_patch_pa, GSP_PHYS_RMCALL_VA_GR_PATCH,
+		boot_stage.rmcall_patch_size, attr, 1, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 2,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_BUNDLE_CB,
+		0, bk + 0x2000ULL, 0, 0, 0, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 3,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PAGEPOOL,
+		0, bk + 0x3000ULL, 0, 0, 0, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 4,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_ATTRIBUTE_CB,
+		0, bk + 0x4000ULL, 0, 0, 0, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 5,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_RTV_CB_GLOBAL,
+		0, bk + 0x5000ULL, 0, 0, 0, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 6,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
+		boot_stage.rmcall_fecs_pa, bk + 0x6000ULL, 0x1000u, attr, 1, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 7,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
+		boot_stage.rmcall_priv_pa, bk + 0x7000ULL, 0x1000u, attr, 1, 0);
+	gsp_phys_rmcall_promote_entry_fill(p, 8,
+		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
+		boot_stage.rmcall_unrestricted_pa, bk + 0x8000ULL,
+		0x1000u, attr, 1, 0);
+
+	for (i = 0; i < 9; i++)
+		pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9 entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s\n",
+			i, le16_to_cpu(p->entry[i].buffer_id),
+			le64_to_cpu(p->entry[i].gpu_phys_addr),
+			le64_to_cpu(p->entry[i].gpu_virt_addr),
+			le64_to_cpu(p->entry[i].size),
+			le32_to_cpu(p->entry[i].phys_attr),
+			p->entry[i].b_initialize, p->entry[i].b_nonmapped,
+			(i == 2 || i == 3 || i == 4 || i == 5) ?
+				" - VA-only booking, no PA" : "");
+
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
+			GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+			p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
+			"GPU_PROMOTE_CTX (golden9, 9 entries, atomic)");
+	pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9: entryCount=9 status=0x%x rpc_result=0x%x (hObject=golden channel 0x%x hChanClient=0x%x engineType=0x%x MAIN pa=0x%llx va=0x%llx size=0x%x; gr_promote_param=%u)\n",
+		status, rpc_result, GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+		boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0,
+		boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
+		boot_stage.rmcall_ctx_size,
+		GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9);
+	if (ret) {
+		pr_err("vgsp-boot: rm-call: PROMOTE_CTX golden9 (9 entries) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the golden context is not committed and the ladder aborts with no retry and no follow-up promote\n",
+		       status, rpc_result,
+		       gsp_phys_rmcall_status_str(status), ret);
+		kvfree(p);
+		return ret;
+	}
+	boot_stage.rmcall_gr_promote_step = 1;
+	kvfree(p);
+	return 0;
+}
+
 /*
  * Fill one MAIN promote entry with the prepared PA/VA and the snapshot's
  * entry size. initialize/nonmapped carry the bisection shape: modes 1/3/5
@@ -8676,6 +8966,11 @@ static int gsp_phys_rmcall_alloc_gr_hvirt(void __iomem *bar0)
  *     entry in the combined kgrobjPromoteContext form where the ladder has
  *     a VA; PM and PRIV_ACCESS_MAP stay init-only and unsourceable buffers
  *     are omitted, all logged per entry.
+ *   - mode 25: the same follow-on FWCLIENT promote on the ladder's V6
+ *     channel, after gsp_phys_rmcall_golden_replay replayed the real golden
+ *     construct (captured VAS control, captured channel shape and the single
+ *     atomic 9-entry PROMOTE_CTX on the golden channel) so the server-side
+ *     GR-instance state exists before this control runs.
  * Mode 0 never reaches this function (stage 3 stops before it); the early
  * return is a belt-and-braces no-op. Success means the server installed the
  * context memdesc, added the VA to the engine-context vaList, committed the
@@ -8707,6 +9002,17 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 		return -EINVAL;
 	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OFF)
 		return -EOPNOTSUPP;
+	/*
+	 * Mode 25's golden construct lives in the golden replay
+	 * (gsp_phys_rmcall_golden_replay); without both gates the replay never
+	 * ran, so the follow-on FWCLIENT promote would have no golden-initialized
+	 * GR-instance state to consume. Abort instead of sending it blind.
+	 */
+	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+	    (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) {
+		pr_err("vgsp-boot: rm-call: mode 25 requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n");
+		return -EINVAL;
+	}
 	/*
 	 * Modes 8/9/10 must run on the V6 channel: without the
 	 * SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping
@@ -8726,7 +9032,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
-	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) &&
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) &&
 	    boot_stage.rmcall_channel_variant !=
 			gsp_phys_rmcall_channel_variant_skipctx.id) {
 		pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
@@ -9406,7 +9713,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 			return ret;
 		}
 		boot_stage.rmcall_gr_promote_step = 1;
-	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) {
+	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) {
 		/*
 		 * Mode 24: the reference FWCLIENT non-3D set as ONE control.
 		 * Order and bufferIds follow kgrobjGetPromoteIds_FWCLIENT
@@ -9423,7 +9731,9 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 		 * stock shape is init-only too; both deviations are logged
 		 * per entry. Auxiliary buffers that cannot be sourced are
 		 * omitted and logged; entryCount is the number of real
-		 * entries.
+		 * entries. Mode 25 shares this follow-on promote on the
+		 * ladder's V6 channel after its golden construct; the
+		 * existing mask selects its entries too (bits 0-7).
 		 */
 		u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
 		u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
@@ -10236,15 +10546,25 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 	 * replay the stock client's post-load golden channel allocations after
 	 * the ladder's VAS/TSG exist and before the ladder's own channel/GR
 	 * object, so the server-side GR-instance state the GR engine-context
-	 * commit needs is initialized first. The first failure aborts the whole
-	 * ladder with no retry; with the parameter 0 this block is skipped
-	 * entirely (no RPC, no log).
+	 * commit needs is initialized first. With mode 25 the replay carries
+	 * the single atomic 9-entry promote and is no longer best-effort: a
+	 * failure aborts the whole ladder with no retry and no follow-up
+	 * promote. For every other mode the replay keeps its historical
+	 * best-effort continuation.
 	 */
 	if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) {
 		ret = gsp_phys_rmcall_golden_replay(bar0);
-		if (ret)
+		if (ret) {
+			if (boot_stage.rmcall_gr_promote ==
+					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+			    gsp_phys_rmcall_engine_is_gr()) {
+				pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-25 golden replay failed (%d); the golden construct is atomic and is not retried\n",
+				       ret);
+				return ret;
+			}
 			pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n",
 				ret);
+		}
 	}
 
 	/*
@@ -14500,6 +14820,11 @@ static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry_count) == 40
 	      "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.entryCount must be at offset 40");
 static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry) == 48,
 	      "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.promoteEntry must be at offset 48 (8-byte aligned)");
+static_assert(sizeof(struct gsp_phys_rmcall_vas_copy_pdes_params) == 184,
+	      "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS must be 184 bytes");
+static_assert(offsetof(struct gsp_phys_rmcall_vas_copy_pdes_params,
+		       num_levels_to_copy) == 32,
+	      "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS.numLevelsToCopy must be at offset 32");
 static_assert(sizeof(struct gsp_phys_rmcall_evict_ctx_params) == 20,
 	      "NV2080_CTRL_GPU_EVICT_CTX_PARAMS must be 20 bytes");
 static_assert(sizeof(struct gsp_phys_rmcall_ctx_buffer_info) == 80,
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 5e6b264..21313a2 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -511,8 +511,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val,
 		return ret;
 
 	if (gsp_phys_rmcall_gr_promote_param < 0 ||
-	    gsp_phys_rmcall_gr_promote_param > 24) {
-		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..24; clamping to 0 (no PROMOTE_CTX)\n",
+	    gsp_phys_rmcall_gr_promote_param > 25) {
+		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..25; clamping to 0 (no PROMOTE_CTX)\n",
 			gsp_phys_rmcall_gr_promote_param);
 		gsp_phys_rmcall_gr_promote_param = 0;
 	}
@@ -531,7 +531,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote,
 		&gsp_phys_rmcall_gr_promote_ops,
 		&gsp_phys_rmcall_gr_promote_param, 0644);
 MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote,
-		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); values outside 0..24 clamp to 0 with a warning");
+		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; values outside 0..25 clamp to 0 with a warning");
 
 int gsp_phys_rmcall_gr_promote_value(void)
 {
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 9e809e3..a38067a 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -250,8 +250,12 @@ int gsp_phys_rmcall_engine_value(void);
  *       PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) as one control on the
  *       V6 KMD-managed channel: combined PA+VA entries where the ladder has
  *       a VA, PM/PRIV_ACCESS_MAP init-only, unsourceable buffers omitted
- *       and logged.
- * The parameter's set callback clamps any value outside 0..24 to 0 with a
+ *       and logged;
+ *   25 = the real golden-client construct on the golden replay handles
+ *       (captured channel shape + single atomic 9-entry PROMOTE_CTX on the
+ *       golden channel) followed by the V6 FWCLIENT promote; requires
+ *       gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1.
+ * The parameter's set callback clamps any value outside 0..25 to 0 with a
  * warning and warns on every nonzero enable (a failed promote can assert
  * GSP-RM and wedge the device; recovery needs a reboot with human approval).
  * The booter snapshots the value when the rm-call ladder's stage 3 starts
-- 
cgit v1.2.3


From e9799396d6a20685a632ddf876d52eb4e127d799 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 18:29:48 -0400
Subject: vgsp: golden9c - bind golden channel to the ladder's proven VAS

The captured golden VAS cannot be fully constructed by the ladder: its
COPY_SERVER_RESERVED_PDES (0x90f10106) consumes client-RM-reserved page-table
level PAs that only exist in the real client's memmgr (and are boot-specific).
Bind the golden channel to the ladder's own FERMI_VASPACE_A, which already has
server page tables installed by the DMA_SET_PAGE_DIRECTORY + PDB build, so the
pVAS validation in kchannelConstruct_IMPL is satisfied. hObjectBuffer stays 0
(host sysmem PB is client-local) and the USERD keep the proven V5 memdesc
pairing.
---
 kernel/gsp_phys_booter.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index a25a0f0..cc95175 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -7917,7 +7917,24 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
 	    gsp_phys_rmcall_engine_is_gr()) {
 		ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9);
+		ch.internal_flags = cpu_to_le32(
+			GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL);
+		/*
+		 * The captured golden VAS has no server page tables until the
+		 * 0x90f10106 COPY_SERVER_RESERVED_PDES control runs, and the
+		 * channel alloc's pVAS validation rejects an empty VAS (NOCAT
+		 * 0x51a8f10/0x52008f4/0x51a89a4). Bind the golden channel to
+		 * the ladder's own VAS (the one the working V5 channel/TSG
+		 * already uses, with its page tables installed).
+		 */
+		ch.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
 		ch.h_object_buffer = cpu_to_le32(0);
+		/*
+		 * The real wire carries its client-local USERD handle; the
+		 * ladder keeps the memdesc pair the V5 shape proved
+		 * (userdMem sysmem chunk, handle array zeroed).
+		 */
+		ch.h_userd_memory[0] = cpu_to_le32(0);
 		ch.instance_mem = gsp_phys_rmcall_mem_desc(
 			GSP_PHYS_RMCALL_BUF_INSTANCE);
 		ch.userd_mem = gsp_phys_rmcall_mem_desc(
-- 
cgit v1.2.3


From cfa2a54775396fd732afc36af65624b11d37c095 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 18:57:27 -0400
Subject: vgsp: golden9d - FBMEM channel memdescs and FBMEM MAIN

The golden9c cycle reached the promote commit and hung the GR engine in the
ctxsw wait (FECS_A/B TDR records then a fatal assert). The real golden capture
carried client-local FBMEM instance/USERD/RAMFC/mthd memdescs and FBMEM ctx
buffers; the host-sysmem substitution is the remaining aperture difference.
Allocate one FBMEM block for the four channel memdescs (sliced 0x1000/0x200/
0x1000/0x1000 like the client memmgr) and back the promoted MAIN with the
server FBMEM allocation (VIDMEM physAttr 0x4, VIDMEM PDB leaf PTEs) for mode
25. PATCH/FECS/PRIV/UNR stay host sysmem in this iteration.
---
 kernel/gsp_phys_booter.c | 77 ++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 61 insertions(+), 16 deletions(-)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index cc95175..36a71d0 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -790,6 +790,18 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL	0xcaf00054u
 #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ	0xcaf00058u
 #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER	0xcaf0005cu
+/*
+ * Mode-25 golden9d: one FBMEM block sliced into the golden channel's
+ * instance/USERD/RAMFC/mthd memdescs (the real capture used client-local
+ * FBMEM for all four; host sysmem made the engine's instance/RAMFC fetch
+ * hang during the ctxsw wait).
+ */
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_MEM	0xcaf00060u
+#define GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE	0x4000u
+#define GSP_PHYS_RMCALL_GOLDEN_FB_INSTANCE	0x0000u
+#define GSP_PHYS_RMCALL_GOLDEN_FB_USERD		0x1000u
+#define GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC		0x2000u
+#define GSP_PHYS_RMCALL_GOLDEN_FB_MTHD		0x3000u
 
 /*
  * Golden-image replay sizes (kernel_graphics.c:2058-2060, 2273-2276): the
@@ -7146,6 +7158,18 @@ gsp_phys_rmcall_mem_desc(u32 index)
 	return d;
 }
 
+static struct gsp_phys_rmcall_mem_desc
+gsp_phys_rmcall_mem_desc_fb(u64 base, u64 size)
+{
+	struct gsp_phys_rmcall_mem_desc d;
+
+	d.base = cpu_to_le64(base);
+	d.size = cpu_to_le64(size);
+	d.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM);
+	d.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED);
+	return d;
+}
+
 /*
  * Channel-alloc variants tried in table order after the TSG exists. V5
  * (tsg-vas-pdb-skip-scrubber) is tried first: the pre-allocated
@@ -7810,6 +7834,7 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	struct gsp_phys_rmcall_channel_params ch;
 	struct gsp_phys_rmcall_mem_obj_shape shape;
 	u32 rpc = 0, status = 0;
+	u64 golden_fb = 0;
 	bool pb_ok = true;
 	int ret;
 
@@ -7930,22 +7955,35 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 		ch.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
 		ch.h_object_buffer = cpu_to_le32(0);
 		/*
-		 * The real wire carries its client-local USERD handle; the
-		 * ladder keeps the memdesc pair the V5 shape proved
-		 * (userdMem sysmem chunk, handle array zeroed).
+		 * The captured golden wire carried client-local FBMEM
+		 * instance/USERD/RAMFC/mthd memdescs; the earlier host-sysmem
+		 * substitution made the engine's instance/RAMFC fetch hang in
+		 * the commit's ctxsw wait (FECS TDRs + fatal assert). Allocate
+		 * one FBMEM block and slice it like the client memmgr would.
+		 * hUserdMemory[0] stays 0 (the real wire's handle is
+		 * client-local and unavailable here).
 		 */
 		ch.h_userd_memory[0] = cpu_to_le32(0);
-		ch.instance_mem = gsp_phys_rmcall_mem_desc(
-			GSP_PHYS_RMCALL_BUF_INSTANCE);
-		ch.userd_mem = gsp_phys_rmcall_mem_desc(
-			GSP_PHYS_RMCALL_BUF_USERD);
-		ch.ramfc_mem = gsp_phys_rmcall_mem_desc(
-			GSP_PHYS_RMCALL_BUF_RAMFC);
-		ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc(
-			GSP_PHYS_RMCALL_BUF_MTHDBUF);
-		pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=host sysmem memdescs (capture used client-local FBMEM)\n",
+		ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+				GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_MEM,
+				GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE, &golden_fb,
+				"GOLDEN channel memdescs");
+		if (ret) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9 channel FBMEM memdesc block (0x%x bytes) allocation failed: %d; the golden replay aborts with no retry\n",
+			       GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE, ret);
+			return ret;
+		}
+		ch.instance_mem = gsp_phys_rmcall_mem_desc_fb(
+			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_INSTANCE, 0x1000);
+		ch.userd_mem = gsp_phys_rmcall_mem_desc_fb(
+			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_USERD, 0x200);
+		ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb(
+			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC, 0x1000);
+		ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc_fb(
+			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_MTHD, 0x1000);
+		pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=FBMEM memdescs block=0x%llx (capture-matched apertures)\n",
 			GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
-			GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES);
+			GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES, golden_fb);
 	}
 	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
 				    boot_stage.rmcall_device,
@@ -8430,7 +8468,9 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0)
 	    boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB ||
 	    boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB) {
+			GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB ||
+	    boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) {
 		u64 fb_base = 0;
 
 		ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
@@ -8788,7 +8828,9 @@ static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
 	gsp_phys_rmcall_promote_entry_fill(p, 0,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
 		boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
-		boot_stage.rmcall_ctx_size, attr, 1, 0);
+		boot_stage.rmcall_ctx_size,
+		boot_stage.rmcall_ctx_fbmem ?
+			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : attr, 1, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 1,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
 		boot_stage.rmcall_patch_pa, GSP_PHYS_RMCALL_VA_GR_PATCH,
@@ -9835,7 +9877,10 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 					GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
 					boot_stage.rmcall_ctx_pa,
 					boot_stage.rmcall_ctx_va,
-					boot_stage.rmcall_ctx_size, attr, 1, 0);
+					boot_stage.rmcall_ctx_size,
+					boot_stage.rmcall_ctx_fbmem ?
+						GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB :
+						attr, 1, 0);
 				gsp_phys_rmcall_gr_promote_log_entry(0,
 					&p->entry[0],
 					"combined (PDB-mapped ctx VA)");
-- 
cgit v1.2.3


From c7f5f7116a2a7e38506cf3c9ea2d5346cc3d681a Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 19:34:47 -0400
Subject: vgsp: golden9e - all ctx buffers FBMEM and every promoted VA
 PDB-mapped

The golden9c cycle reached the server context init and then the GR ctxsw
timed out (FECS_A/B TDR) with host-sysmem MAIN/PATCH/FECS/PRIV/UNR and
unmapped promoted VAs; the golden9d FBMEM test ran on a boot left dirty
by golden9c and its 0x1f is not attributable. The real golden wire carried
FBMEM (physAttr=4) for every ctx buffer, so:
- allocate PATCH/FECS/PRIV/UNR as server FBMEM (PATCH sized by the engine
  geometry query) in gr_ctx_prepare before the PDB build (mode 25 only);
- map PATCH/FECS/PRIV/UNR at their promoted VAs with VIDMEM PTEs in the
  same 2 MB window as MAIN;
- move the FECS/PRIV/UNR and VA-only 3D global VAs into that window and
  give the PA-backed entries PHYS_ATTR_FB.
golden9d's FBMEM channel memdescs are kept.
---
 kernel/gsp_phys_booter.c | 195 ++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 183 insertions(+), 12 deletions(-)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 36a71d0..daf0714 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -390,6 +390,29 @@ bool gsp_phys_boot_requested(void)
  * 32 MB but the live server MAIN is 0x133000; PATCH lives at +0x140000 to
  * stay clear of MAIN's tail). */
 #define GSP_PHYS_RMCALL_VA_GR_PATCH			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x140000ULL)
+/*
+ * Golden9e (mode 25): the promoted FECS/PRIV/UNR buffers and the VA-only
+ * 3D globals are placed inside the same 2 MB window as MAIN/PATCH so every
+ * promoted VA is backed by the ctx PDB page tables (the real golden client
+ * mapped all of them in its VAS; the earlier mode-25 replay wrote unmapped
+ * VAs, which can fault the engine context load).
+ */
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_FECS			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x150000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x151000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_UNR			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x152000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x153000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x154000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x155000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL	(GSP_PHYS_RMCALL_VA_GR_CTX + 0x156000ULL)
+/*
+ * Golden9e server FBMEM allocations for the small promoted buffers (PATCH is
+ * sized by the PATCH engine geometry query; FECS/PRIV/UNR are one 4 KB page
+ * each). The real golden wire carried all of them in FBMEM.
+ */
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PATCH		0xcaf00064u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS		0xcaf00068u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV		0xcaf0006cu
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR		0xcaf00070u
 
 /* GR context-promote bisection modes (gsp_phys_rmcall_gr_promote). */
 #define GSP_PHYS_RMCALL_GR_PROMOTE_OFF			0u
@@ -1483,6 +1506,8 @@ static struct gsp_phys_boot_stage {
 	bool rmcall_ctx_valid;
 	/* mode 11: the context buffer is a server-owned FBMEM allocation */
 	bool rmcall_ctx_fbmem;
+	/* mode 25 golden9e: PATCH/FECS/PRIV/UNR are server-owned FBMEM too */
+	bool rmcall_golden_fb_aux;
 	u64 rmcall_ctx_pa;		/* context buffer DMA base */
 	u64 rmcall_ctx_va;		/* promoted GPU VA window base */
 	u32 rmcall_ctx_size;		/* promoted total size */
@@ -2175,6 +2200,7 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_ctx_prepared = false;
 	boot_stage.rmcall_ctx_valid = false;
 	boot_stage.rmcall_ctx_fbmem = false;
+	boot_stage.rmcall_golden_fb_aux = false;
 	boot_stage.rmcall_ctx_pa = 0;
 	boot_stage.rmcall_ctx_va = 0;
 	boot_stage.rmcall_ctx_size = 0;
@@ -8280,6 +8306,55 @@ static void gsp_phys_rmcall_pdb_build(void)
 						   &boot_stage.rmcall_ctx_pt[pt_page],
 						   0) + (size_t)pt_slot * 8);
 		}
+		/*
+		 * Golden9e: map the FBMEM PATCH/FECS/PRIV/UNR buffers at their
+		 * promoted VAs in the same 2 MB window as MAIN (window 17) with
+		 * VIDMEM PTEs. Every slot used here is above MAIN's tail
+		 * (0x2200000 + 0x133000 = slot 0x133) and distinct.
+		 */
+		if (boot_stage.rmcall_golden_fb_aux) {
+			u8 *ptv = gsp_phys_chunk_page_va(
+				&boot_stage.rmcall_ctx_pt[0], 0);
+			u32 slot, npages;
+
+			npages = DIV_ROUND_UP(boot_stage.rmcall_patch_size,
+					      GSP_PHYS_PAGE_SIZE);
+			slot = (u32)((GSP_PHYS_RMCALL_VA_GR_PATCH >> 12) &
+				     0x1ffu);
+			for (i = 0; i < npages; i++)
+				put_unaligned_le64(
+					gsp_phys_rmcall_pte_vid(
+						boot_stage.rmcall_patch_pa +
+						(u64)i * GSP_PHYS_PAGE_SIZE),
+					ptv + (size_t)(slot + i) * 8);
+			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_FECS >> 12) &
+				     0x1ffu);
+			put_unaligned_le64(
+				gsp_phys_rmcall_pte_vid(
+					boot_stage.rmcall_fecs_pa),
+				ptv + (size_t)slot * 8);
+			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV >> 12) &
+				     0x1ffu);
+			put_unaligned_le64(
+				gsp_phys_rmcall_pte_vid(
+					boot_stage.rmcall_priv_pa),
+				ptv + (size_t)slot * 8);
+			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_UNR >> 12) &
+				     0x1ffu);
+			put_unaligned_le64(
+				gsp_phys_rmcall_pte_vid(
+					boot_stage.rmcall_unrestricted_pa),
+				ptv + (size_t)slot * 8);
+			pr_info("vgsp-boot: rm-call: GOLDEN9E PDB aux mapped: patch va=0x%llx pages=%u (pa=0x%llx) fecs va=0x%llx (pa=0x%llx) priv va=0x%llx (pa=0x%llx) unr va=0x%llx (pa=0x%llx) (VIDMEM PTEs)\n",
+				GSP_PHYS_RMCALL_VA_GR_PATCH, npages,
+				boot_stage.rmcall_patch_pa,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS,
+				boot_stage.rmcall_fecs_pa,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
+				boot_stage.rmcall_priv_pa,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
+				boot_stage.rmcall_unrestricted_pa);
+		}
 		pr_info("vgsp-boot: rm-call: GR ctx PDB mapped: va=0x%llx size=0x%x pages=%u ptPages=%u pd0Entries=%u..%u (%s; fresh slots above the scratch pool; no fetched line rewritten)\n",
 			GSP_PHYS_RMCALL_VA_GR_CTX, boot_stage.rmcall_ctx_size,
 			boot_stage.rmcall_ctx_pages,
@@ -8337,6 +8412,82 @@ static void gsp_phys_rmcall_pdb_build(void)
 			GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
 }
 
+/*
+ * Mode-25 golden9e: allocate the promoted PATCH/FECS/PRIV/UNR context
+ * buffers in server FBMEM (the real golden wire carried FBMEM for every
+ * context buffer; the earlier host-sysmem substitution is a documented
+ * deviation that leaves the engine's context load reading host memory).
+ * PATCH is sized by the PATCH engine geometry query (engineId 0x10); the
+ * three global buffers are one 4 KB page each. Runs before
+ * gsp_phys_rmcall_pdb_build() so the PDB can map every promoted VA in the
+ * same 2 MB window as MAIN. Idempotent; a failure aborts the ladder.
+ */
+static int gsp_phys_rmcall_golden9_fb_aux_prepare(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_engine_ctx_props props;
+	u32 raw, rpc_result = 0, status = 0;
+	u64 fb = 0;
+	int ret;
+
+	if (boot_stage.rmcall_golden_fb_aux)
+		return 0;
+
+	memset(&props, 0, sizeof(props));
+	props.engine_id = cpu_to_le32(GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID);
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
+			GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES,
+			&props, sizeof(props), 24 + sizeof(props),
+			&props, sizeof(props), &rpc_result, &status,
+			"GR_GET_ENGINE_CONTEXT_PROPERTIES (golden9e PATCH)");
+	raw = le32_to_cpu(props.size);
+	pr_info("vgsp-boot: rm-call: GOLDEN9E PATCH geometry: engineId=0x%x rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x\n",
+		GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID, raw,
+		le32_to_cpu(props.alignment), status, rpc_result);
+	if (ret)
+		return ret;
+	if (!raw || raw > GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) {
+		pr_err("vgsp-boot: rm-call: GOLDEN9E PATCH size 0x%x is outside the sanity bound; aborting the ladder with no retry\n",
+		       raw);
+		return -E2BIG;
+	}
+	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PATCH, raw, &fb,
+			"GOLDEN9E PATCH");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_patch_pa = fb;
+	boot_stage.rmcall_patch_size = raw;
+	boot_stage.rmcall_patch_prepared = true;
+
+	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS, 0x1000, &fb,
+			"GOLDEN9E FECS_EVENT");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_fecs_pa = fb;
+
+	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV, 0x1000, &fb,
+			"GOLDEN9E PRIV_ACCESS_MAP");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_priv_pa = fb;
+
+	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR, 0x1000, &fb,
+			"GOLDEN9E UNR_PRIV_ACCESS_MAP");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_unrestricted_pa = fb;
+
+	boot_stage.rmcall_golden_fb_aux = true;
+	pr_info("vgsp-boot: rm-call: GOLDEN9E FBMEM aux buffers ready: patch pa=0x%llx size=0x%x fecs=0x%llx priv=0x%llx unr=0x%llx (server FBMEM, all mapped in the ctx PDB window)\n",
+		boot_stage.rmcall_patch_pa, boot_stage.rmcall_patch_size,
+		boot_stage.rmcall_fecs_pa, boot_stage.rmcall_priv_pa,
+		boot_stage.rmcall_unrestricted_pa);
+	return 0;
+}
+
 /*
  * H1 GR context bind, step 1-2: query the engine-context geometry, then
  * allocate the zeroed, physically contiguous sysmem context buffer the
@@ -8522,6 +8673,19 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0)
 		boot_stage.rmcall_ctx_fbmem ?
 			"server FBMEM, VIDMEM aperture PTEs" :
 			"host sysmem, contiguous, zeroed");
+	/*
+	 * Mode-25 golden9e: prepare the FBMEM PATCH/FECS/PRIV/UNR buffers now,
+	 * before the PDB build maps their promoted VAs.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) {
+		ret = gsp_phys_rmcall_golden9_fb_aux_prepare(bar0);
+		if (ret) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9E FBMEM aux prepare failed (%d); aborting the ladder with no retry\n",
+			       ret);
+			return ret;
+		}
+	}
 	return 0;
 
 err_free:
@@ -8787,8 +8951,8 @@ static void gsp_phys_rmcall_gr_fwclient_small_prepare(void)
 static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_promote_params *p;
-	u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
 	u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
+	u32 fattr;
 	u32 rpc_result = 0, status = 0;
 	unsigned int i;
 	int ret;
@@ -8800,10 +8964,13 @@ static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
 		       boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size);
 		return -EINVAL;
 	}
-	ret = gsp_phys_rmcall_gr_patch_prepare(bar0);
+	/*
+	 * Golden9e: PATCH/FECS/PRIV/UNR are server FBMEM allocations prepared
+	 * with the MAIN (gr_ctx_prepare, before the PDB build). Idempotent.
+	 */
+	ret = gsp_phys_rmcall_golden9_fb_aux_prepare(bar0);
 	if (ret)
 		return ret;
-	gsp_phys_rmcall_gr_fwclient_small_prepare();
 	if (!boot_stage.rmcall_patch_pa || !boot_stage.rmcall_fecs_pa ||
 	    !boot_stage.rmcall_priv_pa ||
 	    !boot_stage.rmcall_unrestricted_pa) {
@@ -8813,6 +8980,8 @@ static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
 		       boot_stage.rmcall_unrestricted_pa);
 		return -ENOMEM;
 	}
+	fattr = boot_stage.rmcall_golden_fb_aux ?
+		GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : attr;
 
 	p = kvzalloc(sizeof(*p), GFP_KERNEL);
 	if (!p)
@@ -8834,29 +9003,31 @@ static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
 	gsp_phys_rmcall_promote_entry_fill(p, 1,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
 		boot_stage.rmcall_patch_pa, GSP_PHYS_RMCALL_VA_GR_PATCH,
-		boot_stage.rmcall_patch_size, attr, 1, 0);
+		boot_stage.rmcall_patch_size, fattr, 1, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 2,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_BUNDLE_CB,
-		0, bk + 0x2000ULL, 0, 0, 0, 0);
+		0, GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB, 0, 0, 0, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 3,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PAGEPOOL,
-		0, bk + 0x3000ULL, 0, 0, 0, 0);
+		0, GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL, 0, 0, 0, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 4,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_ATTRIBUTE_CB,
-		0, bk + 0x4000ULL, 0, 0, 0, 0);
+		0, GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB, 0, 0, 0, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 5,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_RTV_CB_GLOBAL,
-		0, bk + 0x5000ULL, 0, 0, 0, 0);
+		0, GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL, 0, 0, 0, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 6,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
-		boot_stage.rmcall_fecs_pa, bk + 0x6000ULL, 0x1000u, attr, 1, 0);
+		boot_stage.rmcall_fecs_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_FECS,
+		0x1000u, fattr, 1, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 7,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
-		boot_stage.rmcall_priv_pa, bk + 0x7000ULL, 0x1000u, attr, 1, 0);
+		boot_stage.rmcall_priv_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
+		0x1000u, fattr, 1, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 8,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
-		boot_stage.rmcall_unrestricted_pa, bk + 0x8000ULL,
-		0x1000u, attr, 1, 0);
+		boot_stage.rmcall_unrestricted_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
+		0x1000u, fattr, 1, 0);
 
 	for (i = 0; i < 9; i++)
 		pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9 entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s\n",
-- 
cgit v1.2.3


From 777b962455c9e6e51529a5f056e530a42fc1bd05 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 19:43:20 -0400
Subject: vgsp: golden9f - golden channel on the golden VAS with the client PDB

The golden9e ctxsw TDR persisted with capture-faithful FBMEM buffers and
mapped VAs. Source dig: the ladder VAS is IS_EXTERNALLY_OWNED, and on that
path the server's promote defers channel-context binding to UVM (which the
ladder never does), so the engine ctxsw never becomes usable. The real
golden channel runs on the PTETABLE_HEAP_MANAGED golden VAS (client-owned
page tables, not externally owned), which takes the full client-managed
commit path. golden9a/b only failed because that VAS had no page directory:
install the ladder root PDB on the golden VAS via DMA_SET_PAGE_DIRECTORY
before the channel alloc, and bind the golden channel to it.
---
 kernel/gsp_phys_booter.c | 65 ++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 58 insertions(+), 7 deletions(-)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index daf0714..f7cb150 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -7854,6 +7854,45 @@ static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
 	return ret;
 }
 
+/*
+ * Golden9f: install the client PDB hierarchy on the golden VAS. The golden
+ * VAS is PTETABLE_HEAP_MANAGED (client-owned page tables on this config, no
+ * split range), so the supported client mechanism is
+ * DMA_SET_PAGE_DIRECTORY with the ladder's root PDB (the same tree the
+ * ladder VAS uses; its window maps the promoted ctx VAs). Without this the
+ * channel alloc's pVAS validation rejects the empty golden VAS (the
+ * golden9a/b NOCAT chain 0x51a8f10/0x52008f4/0x51a89a4).
+ */
+static int gsp_phys_rmcall_golden9_vas_pdb(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_pdb_params pdb;
+	u32 pdb_rpc = 0;
+	int ret;
+
+	memset(&pdb, 0, sizeof(pdb));
+	pdb.phys_address = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB], 0));
+	pdb.num_entries = cpu_to_le32(GSP_PHYS_RMCALL_PDB_NUM_ENTRIES);
+	pdb.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH |
+				GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS);
+	pdb.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
+	pdb.ch_id = 0;
+	pdb.sub_device_id = cpu_to_le32(GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID);
+	pdb.pasid = 0;
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
+			GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY,
+			&pdb, sizeof(pdb), 16, NULL, 0,
+			&pdb_rpc, NULL,
+			"DMA_SET_PAGE_DIRECTORY (golden9f golden VAS)");
+	pr_info("vgsp-boot: rm-call: GOLDEN9F golden VAS PDB install: hVas=0x%x root=0x%llx numEntries=%u ret=%d rpc_result=0x%x\n",
+		GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
+		(u64)gsp_phys_chunk_page_dma(
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB],
+			0),
+		le32_to_cpu(pdb.num_entries), ret, pdb_rpc);
+	return ret;
+}
+
 static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_vas_params va;
@@ -7900,6 +7939,16 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 			       ret);
 			return ret;
 		}
+		/*
+		 * Golden9f: give the golden VAS real page tables (the ladder's
+		 * root PDB) before the channel alloc validates its pVAS.
+		 */
+		ret = gsp_phys_rmcall_golden9_vas_pdb(bar0);
+		if (ret) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9F golden VAS PDB install failed (%d); the golden replay aborts with no retry\n",
+			       ret);
+			return ret;
+		}
 	}
 
 	/* 2. GPFIFO pushbuffer, physical (kernel_graphics.c:2202-2222). */
@@ -7971,14 +8020,16 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 		ch.internal_flags = cpu_to_le32(
 			GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL);
 		/*
-		 * The captured golden VAS has no server page tables until the
-		 * 0x90f10106 COPY_SERVER_RESERVED_PDES control runs, and the
-		 * channel alloc's pVAS validation rejects an empty VAS (NOCAT
-		 * 0x51a8f10/0x52008f4/0x51a89a4). Bind the golden channel to
-		 * the ladder's own VAS (the one the working V5 channel/TSG
-		 * already uses, with its page tables installed).
+		 * Golden9f: bind the golden channel to the golden VAS (the
+		 * captured shape), now that its page directory is installed
+		 * above. The earlier golden9c workaround used the ladder VAS;
+		 * that VAS is IS_EXTERNALLY_OWNED, so the server's promote
+		 * defers channel-context binding to UVM and the engine ctxsw
+		 * never becomes usable. The golden VAS is PTETABLE_HEAP_MANAGED
+		 * (client page tables, not externally owned) and takes the full
+		 * client-managed commit path.
 		 */
-		ch.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
+		ch.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
 		ch.h_object_buffer = cpu_to_le32(0);
 		/*
 		 * The captured golden wire carried client-local FBMEM
-- 
cgit v1.2.3


From db7774c297ffd30db7a88ebddcf5d0a2d12c5f33 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 19:53:09 -0400
Subject: vgsp: golden9g - fabricated golden-VAS page-table levels + walker ctx
 window

---
 kernel/gsp_phys_booter.c | 185 +++++++++++++++++++++++++++++++----------------
 1 file changed, 124 insertions(+), 61 deletions(-)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index f7cb150..c7abdaf 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -924,6 +924,15 @@ enum gsp_phys_rmcall_buf {
 	GSP_PHYS_RMCALL_BUF_GUEST_PT13,
 	GSP_PHYS_RMCALL_BUF_GUEST_PT14,
 	GSP_PHYS_RMCALL_BUF_GUEST_PT15,
+	/*
+	 * Golden9g: the three client page-table levels the real golden VAS
+	 * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES
+	 * (0x90f10106): PD2 (0x2f, size 0x20), PD1 (0x26) and PD0 (0x1d),
+	 * chained 0 -> 1 -> 2 with the proven GP10X sysmem PDEs.
+	 */
+	GSP_PHYS_RMCALL_BUF_GOLDEN_PT0,
+	GSP_PHYS_RMCALL_BUF_GOLDEN_PT1,
+	GSP_PHYS_RMCALL_BUF_GOLDEN_PT2,
 	GSP_PHYS_RMCALL_BUF_COUNT
 };
 
@@ -6646,6 +6655,7 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
 				   u32 *out_rpc_result, u32 *out_status,
 				   const char *what);
 static void gsp_phys_rmcall_pdb_build(void);
+static u64 gsp_phys_rmcall_pde(u64 pa);
 
 /*
  * Shared-management VAS: install the ladder's page-table window through the
@@ -7795,44 +7805,138 @@ static void gsp_phys_rmcall_golden_fail(const char *step, u32 handle,
 
 /*
  * Mode-25 VAS construct control (0x90f10106), gated by fwclient mask bit 8.
- * The captured real-client control copied three server-RM-internal page-table
- * levels; the ladder has no safe source for those PAs, so the only form this
- * path can send is the zero-level no-op range (numLevelsToCopy=0) and it does
- * so only when the gate is set. With the default mask the control is skipped
- * and the deviation is logged; this sub-step still needs a live bisection
- * (report open item 3).
+ * golden9g: the real client's VAS construction (gvaspaceReserveSplitVaSpace ->
+ * _gvaspaceReserveVaForClientRm) allocates the three client page-table levels
+ * for the server range [4GB, 4.5GB] and hands their PAs to the server with
+ * this control; the server mirrors them with mmuWalkModifyLevelInstance. The
+ * ladder now fabricates the identical chain in host sysmem (pageShifts
+ * 0x2f/0x26/0x1d, sizes 0x20/0x1000/0x1000, aperture SYS_COH) with the proven
+ * GP10X sysmem PDEs, so the golden VAS has the same server-side walker state
+ * as the real client's. Set gsp_phys_rmcall_fwclient_mask bit 8 (0x100) to
+ * send it; without the bit the control is skipped and the deviation logged.
  */
 static int gsp_phys_rmcall_golden9_vas_control(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_vas_copy_pdes_params *p;
+	u64 pa0, pa1, pa2;
 	u32 rpc_result = 0, status = 0;
 	int ret;
 
 	if (!(gsp_phys_rmcall_fwclient_mask_value() &
 	      GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES)) {
-		pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped: the captured three level PAs are server-RM-internal and the ladder has no safe source (uncertain, report open item 3); set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the zero-level no-op-range form for bisection\n",
+		pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped (set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the fabricated three-level chain)\n",
 			GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES);
 		return 0;
 	}
 
+	/* Build the client page-table chain: PT0(PD2[0]) -> PT1(PD1[0]) -> PT2(PD0). */
+	gsp_phys_chunk_zero(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0]);
+	gsp_phys_chunk_zero(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1]);
+	gsp_phys_chunk_zero(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT2]);
+	pa0 = (u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0], 0);
+	pa1 = (u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1], 0);
+	pa2 = (u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT2], 0);
+	put_unaligned_le64(gsp_phys_rmcall_pde(pa1),
+		(u8 *)gsp_phys_chunk_page_va(
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0], 0));
+	put_unaligned_le64(gsp_phys_rmcall_pde(pa2),
+		(u8 *)gsp_phys_chunk_page_va(
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1], 0));
+
 	p = kvzalloc(sizeof(*p), GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
 	p->page_size = cpu_to_le64(1ULL << 29);	/* GMMU_PD1_VADDR_BIT_LO */
+	p->virt_addr_lo = cpu_to_le64(0x100000000ULL);	/* SPLIT_VAS_SERVER_RM_MANAGED_VA_START */
 	p->virt_addr_hi = cpu_to_le64(0x11fffffffULL);
-	p->num_levels_to_copy = cpu_to_le32(0);
+	p->num_levels_to_copy = cpu_to_le32(3);
+	p->levels[0].phys_addr = cpu_to_le64(pa0);
+	p->levels[0].size = cpu_to_le64(0x20);
+	p->levels[0].aperture = cpu_to_le32(4);	/* GMMU_APERTURE_SYS_COH */
+	p->levels[0].page_shift = cpu_to_le32(0x2f);
+	p->levels[1].phys_addr = cpu_to_le64(pa1);
+	p->levels[1].size = cpu_to_le64(0x1000);
+	p->levels[1].aperture = cpu_to_le32(4);
+	p->levels[1].page_shift = cpu_to_le32(0x26);
+	p->levels[2].phys_addr = cpu_to_le64(pa2);
+	p->levels[2].size = cpu_to_le64(0x1000);
+	p->levels[2].aperture = cpu_to_le32(4);
+	p->levels[2].page_shift = cpu_to_le32(0x1d);
 	/* The captured RPC targets the VAS itself (hObject=0xbaba0042). */
 	ret = gsp_phys_rmcall_control(bar0,
 			GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
 			GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES,
 			p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
-			"VASPACE_COPY_SERVER_RESERVED_PDES (no-op range, gated)");
-	pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES: no-op range numLevels=0 pageSize=0x20000000 virtAddrHi=0x11fffffff status=0x%x rpc_result=0x%x ret=%d\n",
-		status, rpc_result, ret);
+			"VASPACE_COPY_SERVER_RESERVED_PDES (3 fabricated levels)");
+	pr_info("vgsp-boot: rm-call: GOLDEN9G VAS COPY_SERVER_RESERVED_PDES: numLevels=3 pageSize=0x20000000 range=[0x%llx,0x%llx] levels={0x%llx/0x20/0x2f, 0x%llx/0x1000/0x26, 0x%llx/0x1000/0x1d} status=0x%x rpc_result=0x%x ret=%d\n",
+		le64_to_cpu(p->virt_addr_lo), le64_to_cpu(p->virt_addr_hi),
+		pa0, pa1, pa2, status, rpc_result, ret);
 	kvfree(p);
 	return ret;
 }
 
+/*
+ * golden9g: install the golden VAS context window (VA_GR_CTX >> 21) in the
+ * server's own MMU walker via NV0080_CTRL_CMD_DMA_UPDATE_PDE_2, the supported
+ * client map path for a non-external VAS. The PT pages hold the golden9e
+ * VIDMEM leaf PTEs (built by gsp_phys_rmcall_pdb_build) for the FBMEM
+ * MAIN/PATCH/FECS/PRIV/UNR buffers, so the GR engine can fetch the promoted
+ * context VAs through the golden VAS exactly as it did through the ladder
+ * VAS's raw PDB. Gated by fwclient mask bit 8 together with the copy control.
+ */
+static int gsp_phys_rmcall_golden9_vas_windows(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_update_pde2_params p;
+	u32 i;
+	int ret;
+
+	if (!(gsp_phys_rmcall_fwclient_mask_value() &
+	      GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES))
+		return 0;
+	if (!boot_stage.rmcall_ctx_pt_pages) {
+		pr_err("vgsp-boot: rm-call: GOLDEN9G ctx window install: no ctx PT pages built; aborting the golden replay with no retry\n");
+		return -EINVAL;
+	}
+	for (i = 0; i < boot_stage.rmcall_ctx_pt_pages; i++) {
+		u32 ctx_index = (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) + i);
+		u64 ctx_pt_pa = (u64)gsp_phys_chunk_page_dma(
+			&boot_stage.rmcall_ctx_pt[i], 0);
+		u32 rpc_result = 0, status = 0;
+
+		memset(&p, 0, sizeof(p));
+		p.pde_index = cpu_to_le32(ctx_index);
+		p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
+		p.pt[0].phys_addr = cpu_to_le64(ctx_pt_pa);
+		p.pt[0].aperture =
+			cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
+		p.pt[1].aperture =
+			cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
+		p.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
+		p.sub_device_id = 0;
+		ret = gsp_phys_rmcall_control(bar0,
+				boot_stage.rmcall_device,
+				GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
+				&p, sizeof(p), sizeof(p), &p, sizeof(p),
+				&rpc_result, &status,
+				"DMA_UPDATE_PDE_2 (golden9g golden VAS ctx window)");
+		if (ret) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window %u rejected: status=0x%x rpc_result=0x%x ret=%d; aborting the golden replay with no retry\n",
+			       ctx_index, status, rpc_result, ret);
+			return ret;
+		}
+		pr_info("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window accepted: pdeIndex=%u ptPa=0x%llx hVas=0x%x status=0x%x\n",
+			ctx_index, ctx_pt_pa,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS, status);
+	}
+	return 0;
+}
+
 static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0);
 
 static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
@@ -7854,45 +7958,6 @@ static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
 	return ret;
 }
 
-/*
- * Golden9f: install the client PDB hierarchy on the golden VAS. The golden
- * VAS is PTETABLE_HEAP_MANAGED (client-owned page tables on this config, no
- * split range), so the supported client mechanism is
- * DMA_SET_PAGE_DIRECTORY with the ladder's root PDB (the same tree the
- * ladder VAS uses; its window maps the promoted ctx VAs). Without this the
- * channel alloc's pVAS validation rejects the empty golden VAS (the
- * golden9a/b NOCAT chain 0x51a8f10/0x52008f4/0x51a89a4).
- */
-static int gsp_phys_rmcall_golden9_vas_pdb(void __iomem *bar0)
-{
-	struct gsp_phys_rmcall_pdb_params pdb;
-	u32 pdb_rpc = 0;
-	int ret;
-
-	memset(&pdb, 0, sizeof(pdb));
-	pdb.phys_address = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
-		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB], 0));
-	pdb.num_entries = cpu_to_le32(GSP_PHYS_RMCALL_PDB_NUM_ENTRIES);
-	pdb.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH |
-				GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS);
-	pdb.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
-	pdb.ch_id = 0;
-	pdb.sub_device_id = cpu_to_le32(GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID);
-	pdb.pasid = 0;
-	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
-			GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY,
-			&pdb, sizeof(pdb), 16, NULL, 0,
-			&pdb_rpc, NULL,
-			"DMA_SET_PAGE_DIRECTORY (golden9f golden VAS)");
-	pr_info("vgsp-boot: rm-call: GOLDEN9F golden VAS PDB install: hVas=0x%x root=0x%llx numEntries=%u ret=%d rpc_result=0x%x\n",
-		GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
-		(u64)gsp_phys_chunk_page_dma(
-			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB],
-			0),
-		le32_to_cpu(pdb.num_entries), ret, pdb_rpc);
-	return ret;
-}
-
 static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_vas_params va;
@@ -7926,9 +7991,11 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	}
 
 	/*
-	 * Mode 25: the captured VAS construct control follows the VAS alloc.
-	 * It is gated off by default and aborts the replay only when the
-	 * operator enabled the gate and the server rejected it.
+	 * Mode 25: the captured VAS construct control follows the VAS alloc,
+	 * then golden9g installs the context window in the golden VAS's
+	 * server walker. Both are gated off by default (fwclient mask bit 8)
+	 * and abort the replay only when the gate is enabled and the server
+	 * rejects a step.
 	 */
 	if (boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
@@ -7939,13 +8006,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 			       ret);
 			return ret;
 		}
-		/*
-		 * Golden9f: give the golden VAS real page tables (the ladder's
-		 * root PDB) before the channel alloc validates its pVAS.
-		 */
-		ret = gsp_phys_rmcall_golden9_vas_pdb(bar0);
+		ret = gsp_phys_rmcall_golden9_vas_windows(bar0);
 		if (ret) {
-			pr_err("vgsp-boot: rm-call: GOLDEN9F golden VAS PDB install failed (%d); the golden replay aborts with no retry\n",
+			pr_err("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window install failed (%d); the golden replay aborts with no retry\n",
 			       ret);
 			return ret;
 		}
@@ -15152,8 +15215,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296
 	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
 	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
-static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 30,
-	      "the rm-call ladder registers exactly thirty DMA buffers (14 ladder + 16 single-page guest scratch PTs)");
+static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33,
+	      "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)");
 static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
 	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
 	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
-- 
cgit v1.2.3


From 4da11e8c343576485f99d5981dddbe1ceed62a85 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 19:59:15 -0400
Subject: vgsp: golden9i - shared-management ladder VAS binding + walker ctx
 window for the golden channel

---
 kernel/gsp_phys_booter.c | 85 ++++++++++++++++++++++++++++++++++--------------
 1 file changed, 60 insertions(+), 25 deletions(-)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index c7abdaf..75b00d1 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -570,6 +570,14 @@ bool gsp_phys_boot_requested(void)
  * is set; the default mask 0x3f leaves it off (skipped, logged).
  */
 #define GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES	(1u << 8)
+/*
+ * Golden9i: bind the golden channel to the ladder's SHARED_MANAGEMENT VAS
+ * (gsp_phys_rmcall_vas_shared=1) instead of the PTETABLE_HEAP_MANAGED golden
+ * VAS, and pre-install the page-table window on it with DMA_UPDATE_PDE_2
+ * before the golden promote. Bit 9 selects this path; bits 8+9 together are
+ * invalid (the golden VAS controls are skipped when bit 9 is set).
+ */
+#define GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS		(1u << 9)
 #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
 	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)
 
@@ -7991,26 +7999,48 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	}
 
 	/*
-	 * Mode 25: the captured VAS construct control follows the VAS alloc,
-	 * then golden9g installs the context window in the golden VAS's
-	 * server walker. Both are gated off by default (fwclient mask bit 8)
-	 * and abort the replay only when the gate is enabled and the server
-	 * rejects a step.
+	 * Mode 25 / golden9i: bind the golden channel to the ladder's
+	 * SHARED_MANAGEMENT VAS and pre-install the ctx window in its server
+	 * walker (DMA_UPDATE_PDE_2), the supported map path for a non-external
+	 * VAS. Otherwise golden9g sends the captured VAS construct control to
+	 * the PTETABLE_HEAP_MANAGED golden VAS and skips the walker install
+	 * (refused 0x1f on a client-managed VAS).
 	 */
 	if (boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
 	    gsp_phys_rmcall_engine_is_gr()) {
-		ret = gsp_phys_rmcall_golden9_vas_control(bar0);
-		if (ret) {
-			pr_err("vgsp-boot: rm-call: GOLDEN9 VAS control failed (%d); the golden replay aborts with no retry\n",
-			       ret);
-			return ret;
-		}
-		ret = gsp_phys_rmcall_golden9_vas_windows(bar0);
-		if (ret) {
-			pr_err("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window install failed (%d); the golden replay aborts with no retry\n",
-			       ret);
-			return ret;
+		if (gsp_phys_rmcall_fwclient_mask_value() &
+		    GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) {
+			if (!boot_stage.rmcall_vas ||
+			    gsp_phys_rmcall_vas_shared_value() == 0) {
+				pr_err("vgsp-boot: rm-call: GOLDEN9I needs the ladder SHARED_MANAGEMENT VAS (rmcall_vas=0x%x shared=%u); aborting the golden replay with no retry\n",
+				       boot_stage.rmcall_vas,
+				       gsp_phys_rmcall_vas_shared_value());
+				return -EINVAL;
+			}
+			ret = gsp_phys_rmcall_map_pt_window(bar0);
+			if (ret) {
+				pr_err("vgsp-boot: rm-call: GOLDEN9I shared-VAS ctx window install failed (%d); aborting the golden replay with no retry\n",
+				       ret);
+				return ret;
+			}
+		} else {
+			ret = gsp_phys_rmcall_golden9_vas_control(bar0);
+			if (ret) {
+				pr_err("vgsp-boot: rm-call: GOLDEN9 VAS control failed (%d); the golden replay aborts with no retry\n",
+				       ret);
+				return ret;
+			}
+			/*
+			 * The walker install is attempted but non-fatal (0x1f on
+			 * the client-managed golden VAS): the channel-alloc datum
+			 * is still collected and the ladder aborts later if the
+			 * promote cannot proceed.
+			 */
+			ret = gsp_phys_rmcall_golden9_vas_windows(bar0);
+			if (ret)
+				pr_warn("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window install failed (%d); continuing without the walker mapping (the golden VAS controls are non-fatal here)\n",
+					ret);
 		}
 	}
 
@@ -8083,16 +8113,21 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 		ch.internal_flags = cpu_to_le32(
 			GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL);
 		/*
-		 * Golden9f: bind the golden channel to the golden VAS (the
-		 * captured shape), now that its page directory is installed
-		 * above. The earlier golden9c workaround used the ladder VAS;
-		 * that VAS is IS_EXTERNALLY_OWNED, so the server's promote
-		 * defers channel-context binding to UVM and the engine ctxsw
-		 * never becomes usable. The golden VAS is PTETABLE_HEAP_MANAGED
-		 * (client page tables, not externally owned) and takes the full
-		 * client-managed commit path.
+		 * Golden9f/9i VAS binding: the captured shape runs on the
+		 * PTETABLE_HEAP_MANAGED golden VAS; golden9i (mask bit 9)
+		 * instead binds to the ladder's SHARED_MANAGEMENT VAS whose
+		 * server walker takes the ctx-window DMA_UPDATE_PDE_2 install
+		 * (the client-managed golden VAS refuses it 0x1f). The
+		 * IS_EXTERNALLY_OWNED ladder default is the golden9c shape and
+		 * is not used by either golden9 mode.
 		 */
-		ch.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
+		if (gsp_phys_rmcall_fwclient_mask_value() &
+		    GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS)
+			ch.h_va_space =
+				cpu_to_le32(boot_stage.rmcall_vas);
+		else
+			ch.h_va_space =
+				cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
 		ch.h_object_buffer = cpu_to_le32(0);
 		/*
 		 * The captured golden wire carried client-local FBMEM
-- 
cgit v1.2.3


From 52266aa4c59b1b551862cf26d66134300997fd40 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 20:04:05 -0400
Subject: vgsp: golden9j - FBMEM backing + VAS mappings for the four 3D global
 ctx buffers

---
 kernel/gsp_phys_booter.c | 133 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 128 insertions(+), 5 deletions(-)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 75b00d1..6a9542b 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -400,19 +400,29 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_VA_GOLDEN9_FECS			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x150000ULL)
 #define GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x151000ULL)
 #define GSP_PHYS_RMCALL_VA_GOLDEN9_UNR			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x152000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x153000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x154000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x155000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL	(GSP_PHYS_RMCALL_VA_GR_CTX + 0x156000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x160000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x170000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x180000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL	(GSP_PHYS_RMCALL_VA_GR_CTX + 0x190000ULL)
 /*
  * Golden9e server FBMEM allocations for the small promoted buffers (PATCH is
  * sized by the PATCH engine geometry query; FECS/PRIV/UNR are one 4 KB page
- * each). The real golden wire carried all of them in FBMEM.
+ * each). The real golden wire carried all of them in FBMEM. Golden9j adds the
+ * four 3D globals (BUNDLE_CB/PAGEPOOL/ATTRIBUTE_CB/RTV_CB_GLOBAL): the real
+ * client allocated them, promoted their VAs (VA-only entries) and mapped them
+ * in its VAS, so the ctx-init bundle can read/write them; the ladder promoted
+ * the VAs with no backing at all until now.
  */
 #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PATCH		0xcaf00064u
 #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS		0xcaf00068u
 #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV		0xcaf0006cu
 #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR		0xcaf00070u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_BUNDLE		0xcaf00074u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PAGEPOOL	0xcaf00078u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_ATTR		0xcaf0007cu
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_RTV		0xcaf00080u
+#define GSP_PHYS_RMCALL_GR_GLOBAL_SLOT			0x10000u
+#define GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES		16u
 
 /* GR context-promote bisection modes (gsp_phys_rmcall_gr_promote). */
 #define GSP_PHYS_RMCALL_GR_PROMOTE_OFF			0u
@@ -1586,6 +1596,15 @@ static struct gsp_phys_boot_stage {
 	u64 rmcall_fecs_pa;
 	u64 rmcall_priv_pa;
 	u64 rmcall_unrestricted_pa;
+	/* golden9j: FBMEM backing for the four promoted 3D globals */
+	u64 rmcall_bundle_pa;
+	u64 rmcall_pagepool_pa;
+	u64 rmcall_attr_pa;
+	u64 rmcall_rtv_pa;
+	u32 rmcall_bundle_size;
+	u32 rmcall_pagepool_size;
+	u32 rmcall_attr_size;
+	u32 rmcall_rtv_size;
 	/*
 	 * GR context-promote bisection snapshot (stage 3, GR0 only): the 0..3
 	 * gsp_phys_rmcall_gr_promote value, the number of accepted promote
@@ -8494,6 +8513,59 @@ static void gsp_phys_rmcall_pdb_build(void)
 				gsp_phys_rmcall_pte_vid(
 					boot_stage.rmcall_unrestricted_pa),
 				ptv + (size_t)slot * 8);
+			/*
+			 * Golden9j: back and map the four 3D globals at their
+			 * promoted VAs (the promote entries stay VA-only, as
+			 * captured). Each VA slot holds up to 16 pages.
+			 */
+			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB >>
+				      12) & 0x1ffu);
+			npages = min_t(u32,
+				DIV_ROUND_UP(boot_stage.rmcall_bundle_size,
+					     GSP_PHYS_PAGE_SIZE),
+				GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES);
+			for (i = 0; i < npages; i++)
+				put_unaligned_le64(
+					gsp_phys_rmcall_pte_vid(
+						boot_stage.rmcall_bundle_pa +
+						(u64)i * GSP_PHYS_PAGE_SIZE),
+					ptv + (size_t)(slot + i) * 8);
+			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL >>
+				      12) & 0x1ffu);
+			npages = min_t(u32,
+				DIV_ROUND_UP(boot_stage.rmcall_pagepool_size,
+					     GSP_PHYS_PAGE_SIZE),
+				GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES);
+			for (i = 0; i < npages; i++)
+				put_unaligned_le64(
+					gsp_phys_rmcall_pte_vid(
+						boot_stage.rmcall_pagepool_pa +
+						(u64)i * GSP_PHYS_PAGE_SIZE),
+					ptv + (size_t)(slot + i) * 8);
+			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB >>
+				      12) & 0x1ffu);
+			npages = min_t(u32,
+				DIV_ROUND_UP(boot_stage.rmcall_attr_size,
+					     GSP_PHYS_PAGE_SIZE),
+				GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES);
+			for (i = 0; i < npages; i++)
+				put_unaligned_le64(
+					gsp_phys_rmcall_pte_vid(
+						boot_stage.rmcall_attr_pa +
+						(u64)i * GSP_PHYS_PAGE_SIZE),
+					ptv + (size_t)(slot + i) * 8);
+			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL >>
+				      12) & 0x1ffu);
+			npages = min_t(u32,
+				DIV_ROUND_UP(boot_stage.rmcall_rtv_size,
+					     GSP_PHYS_PAGE_SIZE),
+				GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES);
+			for (i = 0; i < npages; i++)
+				put_unaligned_le64(
+					gsp_phys_rmcall_pte_vid(
+						boot_stage.rmcall_rtv_pa +
+						(u64)i * GSP_PHYS_PAGE_SIZE),
+					ptv + (size_t)(slot + i) * 8);
 			pr_info("vgsp-boot: rm-call: GOLDEN9E PDB aux mapped: patch va=0x%llx pages=%u (pa=0x%llx) fecs va=0x%llx (pa=0x%llx) priv va=0x%llx (pa=0x%llx) unr va=0x%llx (pa=0x%llx) (VIDMEM PTEs)\n",
 				GSP_PHYS_RMCALL_VA_GR_PATCH, npages,
 				boot_stage.rmcall_patch_pa,
@@ -8503,6 +8575,15 @@ static void gsp_phys_rmcall_pdb_build(void)
 				boot_stage.rmcall_priv_pa,
 				GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
 				boot_stage.rmcall_unrestricted_pa);
+			pr_info("vgsp-boot: rm-call: GOLDEN9J globals mapped: bundle va=0x%llx (pa=0x%llx) pagepool va=0x%llx (pa=0x%llx) attr va=0x%llx (pa=0x%llx) rtv va=0x%llx (pa=0x%llx) (VIDMEM PTEs, one page each)\n",
+				GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB,
+				boot_stage.rmcall_bundle_pa,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL,
+				boot_stage.rmcall_pagepool_pa,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB,
+				boot_stage.rmcall_attr_pa,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
+				boot_stage.rmcall_rtv_pa);
 		}
 		pr_info("vgsp-boot: rm-call: GR ctx PDB mapped: va=0x%llx size=0x%x pages=%u ptPages=%u pd0Entries=%u..%u (%s; fresh slots above the scratch pool; no fetched line rewritten)\n",
 			GSP_PHYS_RMCALL_VA_GR_CTX, boot_stage.rmcall_ctx_size,
@@ -8629,11 +8710,53 @@ static int gsp_phys_rmcall_golden9_fb_aux_prepare(void __iomem *bar0)
 		return ret;
 	boot_stage.rmcall_unrestricted_pa = fb;
 
+	/*
+	 * Golden9j: the four 3D globals. The real client allocated each one
+	 * with the size its own geometry query returned and mapped it in the
+	 * channel VAS; the promote carries the VAs (bInitialize=0, no PA), so
+	 * the ctx-init bundle reads and writes them through the page tables.
+	 * Sizes are clamped to one 16-page slot per VA constant.
+	 */
+	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_BUNDLE, 0x1000, &fb,
+			"GOLDEN9J BUNDLE_CB");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_bundle_pa = fb;
+	boot_stage.rmcall_bundle_size = 0x1000;
+
+	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PAGEPOOL, 0x1000, &fb,
+			"GOLDEN9J PAGEPOOL");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_pagepool_pa = fb;
+	boot_stage.rmcall_pagepool_size = 0x1000;
+
+	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_ATTR, 0x1000, &fb,
+			"GOLDEN9J ATTRIBUTE_CB");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_attr_pa = fb;
+	boot_stage.rmcall_attr_size = 0x1000;
+
+	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_RTV, 0x1000, &fb,
+			"GOLDEN9J RTV_CB_GLOBAL");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_rtv_pa = fb;
+	boot_stage.rmcall_rtv_size = 0x1000;
+
 	boot_stage.rmcall_golden_fb_aux = true;
 	pr_info("vgsp-boot: rm-call: GOLDEN9E FBMEM aux buffers ready: patch pa=0x%llx size=0x%x fecs=0x%llx priv=0x%llx unr=0x%llx (server FBMEM, all mapped in the ctx PDB window)\n",
 		boot_stage.rmcall_patch_pa, boot_stage.rmcall_patch_size,
 		boot_stage.rmcall_fecs_pa, boot_stage.rmcall_priv_pa,
 		boot_stage.rmcall_unrestricted_pa);
+	pr_info("vgsp-boot: rm-call: GOLDEN9J global buffers ready: bundle=0x%llx pagepool=0x%llx attr=0x%llx rtv=0x%llx (server FBMEM, one page each, mapped at the promoted VAs)\n",
+		boot_stage.rmcall_bundle_pa, boot_stage.rmcall_pagepool_pa,
+		boot_stage.rmcall_attr_pa, boot_stage.rmcall_rtv_pa);
 	return 0;
 }
 
-- 
cgit v1.2.3


From ac3ecc3857efdf5da9dcafeb433dc9af65ae738f Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 5 Oct 2026 20:23:44 -0400
Subject: vgsp: golden9k - size every promoted GR ctx buffer from
 GET_ENGINE_CONTEXT_PROPERTIES (REAL GR SEM OK)

The golden replay promoted 0x1000 placeholders for FECS_EVENT/PRIV/UNR and
1-page VA-only mappings for the four 3D globals; the server's own geometry
reports FECS_EVENT 0x10000, PRIV/UNR 0x80000 each, PAGEPOOL 0x20000,
ATTRIBUTE_CB 0x7d7400 and RTV_CB_GLOBAL 0x80000 on this TU104. The girmap
sizes are wrong for the engine's ctx-init access-map copies and the ctxsw
bundle, producing INJECTED_BUNDLE_ERROR + FECS ctxsw TDR on every VAS shape.

fwclient-mask bit 10 (0x400) sizes every promoted buffer from the server's
GET_ENGINE_CONTEXT_PROPERTIES (0x801707), re-spaces the VAs to per-buffer
caps, extends the ctx small-PT pages to six 2 MB windows and promotes the
real sizes. Result (golden9k2): golden promote status 0, FWCLIENT MAIN
promote status 0, stage 3 complete, REAL GR SEM OK sem=0xc0ffee01
GPGet=1 GPPut=1, clean teardown, no wedge.
---
 kernel/gsp_phys_booter.c | 448 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 329 insertions(+), 119 deletions(-)

(limited to 'kernel')

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 6a9542b..764e261 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -398,12 +398,37 @@ bool gsp_phys_boot_requested(void)
  * VAs, which can fault the engine context load).
  */
 #define GSP_PHYS_RMCALL_VA_GOLDEN9_FECS			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x150000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x151000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_UNR			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x152000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x160000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x170000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x180000ULL)
-#define GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL	(GSP_PHYS_RMCALL_VA_GR_CTX + 0x190000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x160000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_UNR			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x1e0000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x260000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x270000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x2b0000ULL)
+#define GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL	(GSP_PHYS_RMCALL_VA_GR_CTX + 0xab0000ULL)
+/*
+ * Golden9k: every promoted context buffer is allocated with the size
+ * GET_ENGINE_CONTEXT_PROPERTIES reports for its engine (the client's own
+ * source for those sizes is the GSP static info; the same control returns
+ * them on this server). Live TU104 sizes (golden9k cycle, 2026-10-05):
+ * FECS_EVENT 0x10000, PRIV/UNR 0x80000 each, BUNDLE_CB 0x3000, PAGEPOOL
+ * 0x20000, ATTRIBUTE_CB 0x7d7400, RTV_CB_GLOBAL 0x80000. The golden9j
+ * hardcoded 0x1000/1-page shapes cannot hold these (the engine's ctx-init
+ * copy of the register access maps is 512 KB per map; the attribute CB is
+ * ~8 MB), which is the leading explanation for the invariant
+ * INJECTED_BUNDLE_ERROR + FECS ctxsw TDR. The VAs below are spaced by the
+ * per-buffer caps; a reported size above its cap aborts before any promote.
+ */
+#define GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX		0x10000u
+#define GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX		0x80000u
+#define GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX		0x10000u
+#define GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX		0x40000u
+#define GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX		0x800000u
+#define GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX		0x80000u
+#define GSP_PHYS_RMCALL_GR_FECS_EVENT_ENGINE_ID		0x17u
+#define GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_ENGINE_ID	0x18u
+#define GSP_PHYS_RMCALL_GR_BUNDLE_CB_ENGINE_ID		0x11u
+#define GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_ENGINE_ID	0x12u
+#define GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_ENGINE_ID	0x13u
+#define GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_ENGINE_ID	0x14u
 /*
  * Golden9e server FBMEM allocations for the small promoted buffers (PATCH is
  * sized by the PATCH engine geometry query; FECS/PRIV/UNR are one 4 KB page
@@ -588,6 +613,16 @@ bool gsp_phys_boot_requested(void)
  * invalid (the golden VAS controls are skipped when bit 9 is set).
  */
 #define GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS		(1u << 9)
+/*
+ * Golden9k: allocate and promote the golden context buffers at the sizes
+ * GET_ENGINE_CONTEXT_PROPERTIES reports (FECS_EVENT/PRIV_ACCESS_MAP/
+ * UNRESTRICTED_PRIV_ACCESS_MAP/BUNDLE_CB/PAGEPOOL/ATTRIBUTE_CB/RTV_CB_GLOBAL)
+ * instead of the golden9j/j one-page placeholder sizes, and back every
+ * promoted VA with PTEs covering the full size across as many 2 MB windows
+ * as needed. Bit 10 selects this path; the golden9i/j shapes stay available
+ * for comparison.
+ */
+#define GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS	(1u << 10)
 #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
 	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)
 
@@ -1605,6 +1640,10 @@ static struct gsp_phys_boot_stage {
 	u32 rmcall_pagepool_size;
 	u32 rmcall_attr_size;
 	u32 rmcall_rtv_size;
+	/* golden9k: sizes queried from GET_ENGINE_CONTEXT_PROPERTIES */
+	u32 rmcall_fecs_size;
+	u32 rmcall_priv_size;
+	u32 rmcall_unr_size;
 	/*
 	 * GR context-promote bisection snapshot (stage 3, GR0 only): the 0..3
 	 * gsp_phys_rmcall_gr_promote value, the number of accepted promote
@@ -2237,6 +2276,13 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_ctx_valid = false;
 	boot_stage.rmcall_ctx_fbmem = false;
 	boot_stage.rmcall_golden_fb_aux = false;
+	boot_stage.rmcall_fecs_size = 0;
+	boot_stage.rmcall_priv_size = 0;
+	boot_stage.rmcall_unr_size = 0;
+	boot_stage.rmcall_bundle_size = 0;
+	boot_stage.rmcall_pagepool_size = 0;
+	boot_stage.rmcall_attr_size = 0;
+	boot_stage.rmcall_rtv_size = 0;
 	boot_stage.rmcall_ctx_pa = 0;
 	boot_stage.rmcall_ctx_va = 0;
 	boot_stage.rmcall_ctx_size = 0;
@@ -8275,6 +8321,36 @@ static u64 gsp_phys_rmcall_pte_vid(u64 fb_offset)
 	return 0x9ull | ((fb_offset >> 12) << 8);
 }
 
+/*
+ * Golden9k: map [va, va+size) with VIDMEM PTEs into the ctx small-PT pages.
+ * One rmcall_ctx_pt page covers one 2 MB window (windows start at
+ * GSP_PHYS_RMCALL_VA_GR_CTX >> 21); a range may span windows, and the PT
+ * pages for every window it touches must already exist (gr_ctx_prepare plus
+ * the golden9k extension in golden9_fb_aux_prepare). Every slot written is
+ * distinct from MAIN's and from the other aux buffers (the caller places
+ * them on 1 MB slots).
+ */
+static void gsp_phys_rmcall_golden9_map_range(u64 va, u64 pa, u32 size)
+{
+	u32 first_win = (u32)(GSP_PHYS_RMCALL_VA_GR_CTX >> 21);
+	u32 pages = DIV_ROUND_UP(size, GSP_PHYS_PAGE_SIZE);
+	u32 i;
+
+	for (i = 0; i < pages; i++) {
+		u64 v = va + (u64)i * GSP_PHYS_PAGE_SIZE;
+		u32 win = (u32)(v >> 21) - first_win;
+		u32 slot = (u32)((v >> 12) & 0x1ffu);
+		u8 *ptv;
+
+		if (win >= boot_stage.rmcall_ctx_pt_pages)
+			break;
+		ptv = gsp_phys_chunk_page_va(&boot_stage.rmcall_ctx_pt[win], 0);
+		put_unaligned_le64(gsp_phys_rmcall_pte_vid(pa +
+					(u64)i * GSP_PHYS_PAGE_SIZE),
+				   ptv + (size_t)slot * 8);
+	}
+}
+
 /*
  * Build the channel's page-directory hierarchy around the 2 MB VA window
  * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO, pushbuffer and the CE copy's
@@ -8475,114 +8551,70 @@ static void gsp_phys_rmcall_pdb_build(void)
 						   0) + (size_t)pt_slot * 8);
 		}
 		/*
-		 * Golden9e: map the FBMEM PATCH/FECS/PRIV/UNR buffers at their
-		 * promoted VAs in the same 2 MB window as MAIN (window 17) with
-		 * VIDMEM PTEs. Every slot used here is above MAIN's tail
-		 * (0x2200000 + 0x133000 = slot 0x133) and distinct.
+		 * Golden9e/j/k: map the FBMEM PATCH/FECS/PRIV/UNR and the four
+		 * 3D global buffers at their promoted VAs with VIDMEM PTEs.
+		 * Every buffer sits on its own 1 MB slot, so the ranges never
+		 * overlap and can span 2 MB windows (the helper picks the PT
+		 * page per page).
 		 */
 		if (boot_stage.rmcall_golden_fb_aux) {
-			u8 *ptv = gsp_phys_chunk_page_va(
-				&boot_stage.rmcall_ctx_pt[0], 0);
-			u32 slot, npages;
-
-			npages = DIV_ROUND_UP(boot_stage.rmcall_patch_size,
-					      GSP_PHYS_PAGE_SIZE);
-			slot = (u32)((GSP_PHYS_RMCALL_VA_GR_PATCH >> 12) &
-				     0x1ffu);
-			for (i = 0; i < npages; i++)
-				put_unaligned_le64(
-					gsp_phys_rmcall_pte_vid(
-						boot_stage.rmcall_patch_pa +
-						(u64)i * GSP_PHYS_PAGE_SIZE),
-					ptv + (size_t)(slot + i) * 8);
-			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_FECS >> 12) &
-				     0x1ffu);
-			put_unaligned_le64(
-				gsp_phys_rmcall_pte_vid(
-					boot_stage.rmcall_fecs_pa),
-				ptv + (size_t)slot * 8);
-			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV >> 12) &
-				     0x1ffu);
-			put_unaligned_le64(
-				gsp_phys_rmcall_pte_vid(
-					boot_stage.rmcall_priv_pa),
-				ptv + (size_t)slot * 8);
-			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_UNR >> 12) &
-				     0x1ffu);
-			put_unaligned_le64(
-				gsp_phys_rmcall_pte_vid(
-					boot_stage.rmcall_unrestricted_pa),
-				ptv + (size_t)slot * 8);
-			/*
-			 * Golden9j: back and map the four 3D globals at their
-			 * promoted VAs (the promote entries stay VA-only, as
-			 * captured). Each VA slot holds up to 16 pages.
-			 */
-			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB >>
-				      12) & 0x1ffu);
-			npages = min_t(u32,
-				DIV_ROUND_UP(boot_stage.rmcall_bundle_size,
-					     GSP_PHYS_PAGE_SIZE),
-				GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES);
-			for (i = 0; i < npages; i++)
-				put_unaligned_le64(
-					gsp_phys_rmcall_pte_vid(
-						boot_stage.rmcall_bundle_pa +
-						(u64)i * GSP_PHYS_PAGE_SIZE),
-					ptv + (size_t)(slot + i) * 8);
-			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL >>
-				      12) & 0x1ffu);
-			npages = min_t(u32,
-				DIV_ROUND_UP(boot_stage.rmcall_pagepool_size,
-					     GSP_PHYS_PAGE_SIZE),
-				GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES);
-			for (i = 0; i < npages; i++)
-				put_unaligned_le64(
-					gsp_phys_rmcall_pte_vid(
-						boot_stage.rmcall_pagepool_pa +
-						(u64)i * GSP_PHYS_PAGE_SIZE),
-					ptv + (size_t)(slot + i) * 8);
-			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB >>
-				      12) & 0x1ffu);
-			npages = min_t(u32,
-				DIV_ROUND_UP(boot_stage.rmcall_attr_size,
-					     GSP_PHYS_PAGE_SIZE),
-				GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES);
-			for (i = 0; i < npages; i++)
-				put_unaligned_le64(
-					gsp_phys_rmcall_pte_vid(
-						boot_stage.rmcall_attr_pa +
-						(u64)i * GSP_PHYS_PAGE_SIZE),
-					ptv + (size_t)(slot + i) * 8);
-			slot = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL >>
-				      12) & 0x1ffu);
-			npages = min_t(u32,
-				DIV_ROUND_UP(boot_stage.rmcall_rtv_size,
-					     GSP_PHYS_PAGE_SIZE),
-				GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES);
-			for (i = 0; i < npages; i++)
-				put_unaligned_le64(
-					gsp_phys_rmcall_pte_vid(
-						boot_stage.rmcall_rtv_pa +
-						(u64)i * GSP_PHYS_PAGE_SIZE),
-					ptv + (size_t)(slot + i) * 8);
-			pr_info("vgsp-boot: rm-call: GOLDEN9E PDB aux mapped: patch va=0x%llx pages=%u (pa=0x%llx) fecs va=0x%llx (pa=0x%llx) priv va=0x%llx (pa=0x%llx) unr va=0x%llx (pa=0x%llx) (VIDMEM PTEs)\n",
-				GSP_PHYS_RMCALL_VA_GR_PATCH, npages,
+			gsp_phys_rmcall_golden9_map_range(
+				GSP_PHYS_RMCALL_VA_GR_PATCH,
+				boot_stage.rmcall_patch_pa,
+				boot_stage.rmcall_patch_size);
+			gsp_phys_rmcall_golden9_map_range(
+				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS,
+				boot_stage.rmcall_fecs_pa,
+				boot_stage.rmcall_fecs_size);
+			gsp_phys_rmcall_golden9_map_range(
+				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
+				boot_stage.rmcall_priv_pa,
+				boot_stage.rmcall_priv_size);
+			gsp_phys_rmcall_golden9_map_range(
+				GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
+				boot_stage.rmcall_unrestricted_pa,
+				boot_stage.rmcall_unr_size);
+			gsp_phys_rmcall_golden9_map_range(
+				GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB,
+				boot_stage.rmcall_bundle_pa,
+				boot_stage.rmcall_bundle_size);
+			gsp_phys_rmcall_golden9_map_range(
+				GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL,
+				boot_stage.rmcall_pagepool_pa,
+				boot_stage.rmcall_pagepool_size);
+			gsp_phys_rmcall_golden9_map_range(
+				GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB,
+				boot_stage.rmcall_attr_pa,
+				boot_stage.rmcall_attr_size);
+			gsp_phys_rmcall_golden9_map_range(
+				GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
+				boot_stage.rmcall_rtv_pa,
+				boot_stage.rmcall_rtv_size);
+			pr_info("vgsp-boot: rm-call: GOLDEN9E PDB aux mapped: patch va=0x%llx size=0x%x (pa=0x%llx) fecs va=0x%llx size=0x%x (pa=0x%llx) priv va=0x%llx size=0x%x (pa=0x%llx) unr va=0x%llx size=0x%x (pa=0x%llx) (VIDMEM PTEs)\n",
+				GSP_PHYS_RMCALL_VA_GR_PATCH,
+				boot_stage.rmcall_patch_size,
 				boot_stage.rmcall_patch_pa,
 				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS,
+				boot_stage.rmcall_fecs_size,
 				boot_stage.rmcall_fecs_pa,
 				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
+				boot_stage.rmcall_priv_size,
 				boot_stage.rmcall_priv_pa,
 				GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
+				boot_stage.rmcall_unr_size,
 				boot_stage.rmcall_unrestricted_pa);
-			pr_info("vgsp-boot: rm-call: GOLDEN9J globals mapped: bundle va=0x%llx (pa=0x%llx) pagepool va=0x%llx (pa=0x%llx) attr va=0x%llx (pa=0x%llx) rtv va=0x%llx (pa=0x%llx) (VIDMEM PTEs, one page each)\n",
+			pr_info("vgsp-boot: rm-call: GOLDEN9J globals mapped: bundle va=0x%llx size=0x%x (pa=0x%llx) pagepool va=0x%llx size=0x%x (pa=0x%llx) attr va=0x%llx size=0x%x (pa=0x%llx) rtv va=0x%llx size=0x%x (pa=0x%llx) (VIDMEM PTEs, full-size coverage)\n",
 				GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB,
+				boot_stage.rmcall_bundle_size,
 				boot_stage.rmcall_bundle_pa,
 				GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL,
+				boot_stage.rmcall_pagepool_size,
 				boot_stage.rmcall_pagepool_pa,
 				GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB,
+				boot_stage.rmcall_attr_size,
 				boot_stage.rmcall_attr_pa,
 				GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
+				boot_stage.rmcall_rtv_size,
 				boot_stage.rmcall_rtv_pa);
 		}
 		pr_info("vgsp-boot: rm-call: GR ctx PDB mapped: va=0x%llx size=0x%x pages=%u ptPages=%u pd0Entries=%u..%u (%s; fresh slots above the scratch pool; no fetched line rewritten)\n",
@@ -8642,6 +8674,42 @@ static void gsp_phys_rmcall_pdb_build(void)
 			GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
 }
 
+/*
+ * Golden9k: query the size/alignment GET_ENGINE_CONTEXT_PROPERTIES reports
+ * for one engine context buffer (the same control the MAIN/PATCH geometry
+ * uses; the server derives the values from the static info the stock client
+ * caches at GSP boot). Returns 0 with size/align set, the RPC error on
+ * failure, -EINVAL on a zero size (buffer unsupported on this engine).
+ */
+static int gsp_phys_rmcall_engine_ctx_size(void __iomem *bar0, u32 engine_id,
+					   const char *name, u32 *size,
+					   u32 *align)
+{
+	struct gsp_phys_rmcall_engine_ctx_props props;
+	u32 rpc_result = 0, status = 0;
+	int ret;
+
+	memset(&props, 0, sizeof(props));
+	props.engine_id = cpu_to_le32(engine_id);
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
+			GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES,
+			&props, sizeof(props), 24 + sizeof(props),
+			&props, sizeof(props), &rpc_result, &status,
+			"GR_GET_ENGINE_CONTEXT_PROPERTIES (golden9k)");
+	*size = le32_to_cpu(props.size);
+	*align = le32_to_cpu(props.alignment);
+	pr_info("vgsp-boot: rm-call: GOLDEN9K %s geometry: engineId=0x%x rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x\n",
+		name, engine_id, *size, *align, status, rpc_result);
+	if (ret)
+		return ret;
+	if (!*size) {
+		pr_err("vgsp-boot: rm-call: GOLDEN9K %s reports size 0 (unsupported); aborting with no promote and no retry\n",
+		       name);
+		return -EINVAL;
+	}
+	return 0;
+}
+
 /*
  * Mode-25 golden9e: allocate the promoted PATCH/FECS/PRIV/UNR context
  * buffers in server FBMEM (the real golden wire carried FBMEM for every
@@ -8656,6 +8724,7 @@ static int gsp_phys_rmcall_golden9_fb_aux_prepare(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_engine_ctx_props props;
 	u32 raw, rpc_result = 0, status = 0;
+	u32 i;
 	u64 fb = 0;
 	int ret;
 
@@ -8689,22 +8758,121 @@ static int gsp_phys_rmcall_golden9_fb_aux_prepare(void __iomem *bar0)
 	boot_stage.rmcall_patch_size = raw;
 	boot_stage.rmcall_patch_prepared = true;
 
+	/*
+	 * Golden9k (fwclient-mask bit 10): size every promoted buffer from the
+	 * engine-context-properties query. The bInitialize=1 entries (FECS_EVENT/
+	 * PRIV/UNR) are written by the server's init copy, and the VA-only
+	 * globals are written by the ctxsw bundle through the page tables; the
+	 * golden9j one-page shapes cannot hold either (the Turing private and
+	 * unrestricted register access maps are 512 KB each). The stock client's
+	 * sizes come from the GSP static info; this control reports the same
+	 * values. A size above its per-buffer VA cap aborts before any promote.
+	 */
+	if (gsp_phys_rmcall_fwclient_mask_value() &
+	    GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS) {
+		u32 align;
+
+		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
+				GSP_PHYS_RMCALL_GR_FECS_EVENT_ENGINE_ID,
+				"FECS_EVENT", &raw, &align);
+		if (ret)
+			return ret;
+		boot_stage.rmcall_fecs_size = raw;
+		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
+				GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_ENGINE_ID,
+				"PRIV_ACCESS_MAP", &raw, &align);
+		if (ret)
+			return ret;
+		boot_stage.rmcall_priv_size = raw;
+		boot_stage.rmcall_unr_size = raw;
+		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
+				GSP_PHYS_RMCALL_GR_BUNDLE_CB_ENGINE_ID,
+				"BUNDLE_CB", &raw, &align);
+		if (ret)
+			return ret;
+		boot_stage.rmcall_bundle_size = raw;
+		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
+				GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_ENGINE_ID,
+				"PAGEPOOL", &raw, &align);
+		if (ret)
+			return ret;
+		boot_stage.rmcall_pagepool_size = raw;
+		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
+				GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_ENGINE_ID,
+				"ATTRIBUTE_CB", &raw, &align);
+		if (ret)
+			return ret;
+		boot_stage.rmcall_attr_size = raw;
+		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
+				GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_ENGINE_ID,
+				"RTV_CB_GLOBAL", &raw, &align);
+		if (ret)
+			return ret;
+		boot_stage.rmcall_rtv_size = raw;
+		if (boot_stage.rmcall_fecs_size >
+				GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX ||
+		    boot_stage.rmcall_priv_size >
+				GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX ||
+		    boot_stage.rmcall_bundle_size >
+				GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX ||
+		    boot_stage.rmcall_pagepool_size >
+				GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX ||
+		    boot_stage.rmcall_attr_size >
+				GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX ||
+		    boot_stage.rmcall_rtv_size >
+				GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9K a promoted buffer exceeds its per-buffer VA cap (fecs=0x%x/0x%x priv=0x%x/0x%x bundle=0x%x/0x%x pagepool=0x%x/0x%x attr=0x%x/0x%x rtv=0x%x/0x%x); aborting with no promote and no retry\n",
+			       boot_stage.rmcall_fecs_size,
+			       GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX,
+			       boot_stage.rmcall_priv_size,
+			       GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX,
+			       boot_stage.rmcall_bundle_size,
+			       GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX,
+			       boot_stage.rmcall_pagepool_size,
+			       GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX,
+			       boot_stage.rmcall_attr_size,
+			       GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX,
+			       boot_stage.rmcall_rtv_size,
+			       GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX);
+			return -E2BIG;
+		}
+		pr_info("vgsp-boot: rm-call: GOLDEN9K sized buffers: fecs=0x%x priv=0x%x unr=0x%x bundle=0x%x pagepool=0x%x attr=0x%x rtv=0x%x (all queried)\n",
+			boot_stage.rmcall_fecs_size,
+			boot_stage.rmcall_priv_size,
+			boot_stage.rmcall_unr_size,
+			boot_stage.rmcall_bundle_size,
+			boot_stage.rmcall_pagepool_size,
+			boot_stage.rmcall_attr_size,
+			boot_stage.rmcall_rtv_size);
+	} else {
+		boot_stage.rmcall_fecs_size = 0x1000;
+		boot_stage.rmcall_priv_size = 0x1000;
+		boot_stage.rmcall_unr_size = 0x1000;
+		boot_stage.rmcall_bundle_size = 0x1000;
+		boot_stage.rmcall_pagepool_size = 0x1000;
+		boot_stage.rmcall_attr_size = 0x1000;
+		boot_stage.rmcall_rtv_size = 0x1000;
+	}
+
 	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
-			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS, 0x1000, &fb,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS,
+			boot_stage.rmcall_fecs_size, &fb,
 			"GOLDEN9E FECS_EVENT");
 	if (ret)
 		return ret;
 	boot_stage.rmcall_fecs_pa = fb;
 
 	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
-			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV, 0x1000, &fb,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV,
+			boot_stage.rmcall_priv_size, &fb,
 			"GOLDEN9E PRIV_ACCESS_MAP");
 	if (ret)
 		return ret;
 	boot_stage.rmcall_priv_pa = fb;
 
 	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
-			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR, 0x1000, &fb,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR,
+			boot_stage.rmcall_unr_size, &fb,
 			"GOLDEN9E UNR_PRIV_ACCESS_MAP");
 	if (ret)
 		return ret;
@@ -8715,48 +8883,90 @@ static int gsp_phys_rmcall_golden9_fb_aux_prepare(void __iomem *bar0)
 	 * with the size its own geometry query returned and mapped it in the
 	 * channel VAS; the promote carries the VAs (bInitialize=0, no PA), so
 	 * the ctx-init bundle reads and writes them through the page tables.
-	 * Sizes are clamped to one 16-page slot per VA constant.
+	 * Golden9k sizes them from the same query as the rest; golden9j kept
+	 * them at one page each.
 	 */
 	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
-			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_BUNDLE, 0x1000, &fb,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_BUNDLE,
+			boot_stage.rmcall_bundle_size, &fb,
 			"GOLDEN9J BUNDLE_CB");
 	if (ret)
 		return ret;
 	boot_stage.rmcall_bundle_pa = fb;
-	boot_stage.rmcall_bundle_size = 0x1000;
 
 	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
-			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PAGEPOOL, 0x1000, &fb,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PAGEPOOL,
+			boot_stage.rmcall_pagepool_size, &fb,
 			"GOLDEN9J PAGEPOOL");
 	if (ret)
 		return ret;
 	boot_stage.rmcall_pagepool_pa = fb;
-	boot_stage.rmcall_pagepool_size = 0x1000;
 
 	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
-			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_ATTR, 0x1000, &fb,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_ATTR,
+			boot_stage.rmcall_attr_size, &fb,
 			"GOLDEN9J ATTRIBUTE_CB");
 	if (ret)
 		return ret;
 	boot_stage.rmcall_attr_pa = fb;
-	boot_stage.rmcall_attr_size = 0x1000;
 
 	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
-			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_RTV, 0x1000, &fb,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_RTV,
+			boot_stage.rmcall_rtv_size, &fb,
 			"GOLDEN9J RTV_CB_GLOBAL");
 	if (ret)
 		return ret;
 	boot_stage.rmcall_rtv_pa = fb;
-	boot_stage.rmcall_rtv_size = 0x1000;
+
+	/*
+	 * Golden9k: the sized layout spans as many 2 MB windows as the highest
+	 * promoted VA needs; allocate the extra small-PT pages now, before the
+	 * PDB build and the DMA_UPDATE_PDE_2 window installs consume
+	 * rmcall_ctx_pt_pages. MAIN's pages were already sized/allocated.
+	 */
+	{
+		u32 first_win = (u32)(GSP_PHYS_RMCALL_VA_GR_CTX >> 21);
+		u32 last_win = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL +
+				      boot_stage.rmcall_rtv_size - 1) >> 21);
+		u32 need = last_win - first_win + 1;
+
+		if (need > GSP_PHYS_RMCALL_GR_CTX_PT_MAX) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9K sized layout needs %u PT pages > max %u; aborting with no promote and no retry\n",
+			       need, GSP_PHYS_RMCALL_GR_CTX_PT_MAX);
+			return -E2BIG;
+		}
+		for (i = boot_stage.rmcall_ctx_pt_pages; i < need; i++) {
+			ret = gsp_phys_chunk_alloc(boot_stage.pdev,
+						   &boot_stage.rmcall_ctx_pt[i],
+						   1u);
+			if (ret) {
+				pr_err("vgsp-boot: rm-call: GOLDEN9K ctx PT page %u/%u allocation failed: %d; aborting the ladder with no retry\n",
+				       i, need, ret);
+				return ret;
+			}
+			gsp_phys_chunk_zero(&boot_stage.rmcall_ctx_pt[i]);
+		}
+		if (need > boot_stage.rmcall_ctx_pt_pages) {
+			pr_info("vgsp-boot: rm-call: GOLDEN9K ctx PT pages extended %u -> %u for the sized layout (last VA 0x%llx + 0x%x)\n",
+				boot_stage.rmcall_ctx_pt_pages, need,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
+				boot_stage.rmcall_rtv_size);
+			boot_stage.rmcall_ctx_pt_pages = need;
+		}
+	}
 
 	boot_stage.rmcall_golden_fb_aux = true;
-	pr_info("vgsp-boot: rm-call: GOLDEN9E FBMEM aux buffers ready: patch pa=0x%llx size=0x%x fecs=0x%llx priv=0x%llx unr=0x%llx (server FBMEM, all mapped in the ctx PDB window)\n",
+	pr_info("vgsp-boot: rm-call: GOLDEN9E FBMEM aux buffers ready: patch pa=0x%llx size=0x%x fecs=0x%llx size=0x%x priv=0x%llx size=0x%x unr=0x%llx size=0x%x (server FBMEM, all mapped in the ctx PDB windows)\n",
 		boot_stage.rmcall_patch_pa, boot_stage.rmcall_patch_size,
-		boot_stage.rmcall_fecs_pa, boot_stage.rmcall_priv_pa,
-		boot_stage.rmcall_unrestricted_pa);
-	pr_info("vgsp-boot: rm-call: GOLDEN9J global buffers ready: bundle=0x%llx pagepool=0x%llx attr=0x%llx rtv=0x%llx (server FBMEM, one page each, mapped at the promoted VAs)\n",
-		boot_stage.rmcall_bundle_pa, boot_stage.rmcall_pagepool_pa,
-		boot_stage.rmcall_attr_pa, boot_stage.rmcall_rtv_pa);
+		boot_stage.rmcall_fecs_pa, boot_stage.rmcall_fecs_size,
+		boot_stage.rmcall_priv_pa, boot_stage.rmcall_priv_size,
+		boot_stage.rmcall_unrestricted_pa,
+		boot_stage.rmcall_unr_size);
+	pr_info("vgsp-boot: rm-call: GOLDEN9J global buffers ready: bundle=0x%llx size=0x%x pagepool=0x%llx size=0x%x attr=0x%llx size=0x%x rtv=0x%llx size=0x%x (server FBMEM, PTEs cover the full size)\n",
+		boot_stage.rmcall_bundle_pa, boot_stage.rmcall_bundle_size,
+		boot_stage.rmcall_pagepool_pa, boot_stage.rmcall_pagepool_size,
+		boot_stage.rmcall_attr_pa, boot_stage.rmcall_attr_size,
+		boot_stage.rmcall_rtv_pa, boot_stage.rmcall_rtv_size);
 	return 0;
 }
 
@@ -9291,15 +9501,15 @@ static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
 	gsp_phys_rmcall_promote_entry_fill(p, 6,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
 		boot_stage.rmcall_fecs_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_FECS,
-		0x1000u, fattr, 1, 0);
+		boot_stage.rmcall_fecs_size, fattr, 1, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 7,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
 		boot_stage.rmcall_priv_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
-		0x1000u, fattr, 1, 0);
+		boot_stage.rmcall_priv_size, fattr, 1, 0);
 	gsp_phys_rmcall_promote_entry_fill(p, 8,
 		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
 		boot_stage.rmcall_unrestricted_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
-		0x1000u, fattr, 1, 0);
+		boot_stage.rmcall_unr_size, fattr, 1, 0);
 
 	for (i = 0; i < 9; i++)
 		pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9 entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s\n",
-- 
cgit v1.2.3