gsp-phys: allocate an externally-owned VAS with a page directory before the channel ladder (V5)

1 files changed, 165 insertions, 39 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index d20ee7e..b5ec52b 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -201,7 +201,9 @@ bool gsp_phys_boot_requested(void)
  *   TURING_DMA_COPY_A 0xC5B5 (class/clc5b5.h:33), NV01_MEMORY_LIST_SYSTEM
  *   0x81 (class/cl84a0.h:45), NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN
  *   0xC36F0108 (ctrl/ctrlc36f.h:82), NVA06F_CTRL_CMD_GPFIFO_SCHEDULE
- *   0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), COPY0 engine type 0x9
+ *   0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), FERMI_VASPACE_A 0x90F1
+ *   (class/cl90f1.h:30), NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY 0x801813
+ *   (ctrl/ctrl0080dma.h:832-840), COPY0 engine type 0x9
  *   (class/cl2080_notification.h:292).
  */
 #define GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY	4u
@@ -215,11 +217,31 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_CLASS_TSG		0xa06cu
 #define GSP_PHYS_RMCALL_CLASS_CHANNEL		0xc46fu
 #define GSP_PHYS_RMCALL_CLASS_CE		0xc5b5u
+#define GSP_PHYS_RMCALL_CLASS_VASPACE		0x90f1u
 #define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN	0xc36f0108u
 #define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE	0xa06f0103u
+#define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY	0x801813u
 #define GSP_PHYS_RMCALL_ENGINE_COPY0		0x9u
 
 /*
+ * FERMI_VASPACE_A allocation (nvos.h:2756-2765): flags bit 3
+ * (IS_EXTERNALLY_OWNED) and bit 8 (ALLOW_ZERO_ADDRESS); the 256 MB window is
+ * the smallest deterministic size this ladder needs and keeps vaBase 0. The
+ * root page directory is committed separately with
+ * NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY (ctrl0080dma.h:832-840), whose flag
+ * bit 1 selects the NONCOHERENT_SYSMEM aperture and bit 3 applies the base to
+ * every channel. subDeviceId follows the RM's ID+1 convention (0 broadcasts).
+ */
+#define GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW		0u
+#define GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED	BIT(3)
+#define GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS	BIT(8)
+#define GSP_PHYS_RMCALL_VAS_SIZE			0x10000000ULL
+#define GSP_PHYS_RMCALL_PDB_NUM_ENTRIES			256u
+#define GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH	BIT(1)
+#define GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS		BIT(3)
+#define GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID		1u
+
+/*
  * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
  * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
  * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
@@ -256,10 +278,12 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_HANDLE_SUBDEVICE	0xcaf00004u
 #define GSP_PHYS_RMCALL_HANDLE_TSG		0xcaf00008u
 #define GSP_PHYS_RMCALL_HANDLE_CHANNEL		0xcaf0000cu
+#define GSP_PHYS_RMCALL_HANDLE_VAS		0xcaf0001cu
 /* Channel-alloc variant handles (see the stage-3 variant walk). */
 #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1	0xcaf00010u
 #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2	0xcaf00014u
 #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3	0xcaf00018u
+#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5	0xcaf00024u
 #define GSP_PHYS_RMCALL_HANDLE_MEM_BASE		0xcaf01000u
 #define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE	4u
 
@@ -291,6 +315,7 @@ enum gsp_phys_rmcall_buf {
 	GSP_PHYS_RMCALL_BUF_INSTANCE,
 	GSP_PHYS_RMCALL_BUF_RAMFC,
 	GSP_PHYS_RMCALL_BUF_MTHDBUF,
+	GSP_PHYS_RMCALL_BUF_VASPDB,
 	GSP_PHYS_RMCALL_BUF_COUNT
 };
 
@@ -822,9 +847,11 @@ static struct gsp_phys_boot_stage {
 	u32 rmcall_client;		/* NV01_ROOT handle (server-chosen) */
 	u32 rmcall_device;
 	u32 rmcall_subdevice;
+	u32 rmcall_vas;			/* explicit FERMI_VASPACE_A handle */
 	u32 rmcall_tsg;
 	u32 rmcall_channel;
-	u32 rmcall_channel_variant;	/* winning stage-3 variant (1..N, 0=none) */
+	u32 rmcall_channel_variant;	/* winning stage-3 variant (0=none; see
+					 * the variant table for its number) */
 	bool rmcall_channel_server_mem;	/* winning variant had server-allocated mem */
 	u32 rmcall_token;		/* GET_WORK_SUBMIT_TOKEN result */
 	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
@@ -1333,6 +1360,7 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_client = 0;
 	boot_stage.rmcall_device = 0;
 	boot_stage.rmcall_subdevice = 0;
+	boot_stage.rmcall_vas = 0;
 	boot_stage.rmcall_tsg = 0;
 	boot_stage.rmcall_channel = 0;
 	boot_stage.rmcall_channel_variant = 0;
@@ -4173,13 +4201,40 @@ struct gsp_phys_rmcall_mem_desc {
 	__le32 cache_attrib;
 };
 
+/* NV_VASPACE_ALLOCATION_PARAMETERS (nvos.h:2756-2765), 48 bytes: index, flags,
+ * vaSize, vaStartInternal, vaLimitInternal, bigPageSize, vaBase. */
+struct gsp_phys_rmcall_vas_params {
+	__le32 index;
+	__le32 flags;
+	__le64 va_size;
+	__le64 va_start_internal;
+	__le64 va_limit_internal;
+	__le32 big_page_size;
+	__le32 pad;
+	__le64 va_base;
+};
+
+/* NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS (ctrl0080dma.h:832-840): the
+ * sysmem physical address of the root page directory, its entry count, the
+ * aperture/applicability flags, the VAS handle, chId and subDeviceId. */
+struct gsp_phys_rmcall_pdb_params {
+	__le64 phys_address;
+	__le32 num_entries;
+	__le32 flags;
+	__le32 h_va_space;
+	__le32 ch_id;
+	__le32 sub_device_id;
+	__le32 pasid;
+};
+
 /* NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS (nvos.h:2511-2518): the TSG's
  * hParent is the device (kernel_channel_group_api.c:150), engineType the
- * group engine (COPY0). hVASpace 0 selects the parent device's default VAS
- * when the device vaMode is not MULTIPLE_VASPACES
- * (kernel_channel_group_api.c:222-233, mem_mgr/vaspace.c:232-242); with a
- * MULTIPLE_VASPACES device an explicit VAS is required and 0 fails with
- * NV_ERR_INVALID_STATE. */
+ * group engine (COPY0). hVASpace carries the explicit FERMI_VASPACE_A whose
+ * page directory was committed before the alloc: the implicit default VAS
+ * (hVASpace 0) has no root page-directory memdesc until a mapping reserves
+ * one, and the channel's context-share/instance-block setup programs that
+ * base and fails NV_ERR_INVALID_STATE (mem_mgr/vaspace.c:232-242,
+ * gpu/mmu/arch/pascal/kern_gmmu_gp100.c:150-152). */
 struct gsp_phys_rmcall_tsg_params {
 	__le32 h_object_error;
 	__le32 h_object_ecc_error;
@@ -4509,7 +4564,7 @@ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index)
 static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
 				   const void *params, u32 params_len,
 				   u32 reply_min, void *out, u32 out_len,
-				   const char *what)
+				   u32 *out_rpc_result, const char *what)
 {
 	struct gsp_phys_rmcall_rm_control_req *req;
 	struct gsp_phys_rx_record rec;
@@ -4529,7 +4584,8 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
 		memcpy(req->params, params, params_len);
 
 	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL,
-				  req, req_len, reply_min, &rec, true, NULL);
+				  req, req_len, reply_min, &rec, true,
+				  out_rpc_result);
 	kvfree(req);
 	if (ret) {
 		pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n",
@@ -4672,7 +4728,7 @@ static int gsp_phys_rmcall_stage2(void __iomem *bar0)
 		}
 	}
 
-	pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf zeroed)\n",
+	pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb zeroed)\n",
 		GSP_PHYS_RMCALL_BUF_COUNT);
 	return 0;
 }
@@ -4691,17 +4747,22 @@ gsp_phys_rmcall_mem_desc(u32 index)
 }
 
 /*
- * Channel-alloc variants tried in order after the TSG exists. V1 mirrors the
- * RM's internal CE channel template (mem_utils_gm107.c:1186-1283: device
- * parent, hDeviceId, SKIP_SCRUBBER flag, PRIVILEGE_KERNEL internalFlags).
- * V2 keeps the reference shape but zeroes the memdescs and GPFIFO so the
- * server allocates channel memory itself (no client ring to fill, so stage 4
- * refuses). V3 is the previously tried control shape (TSG parent, flags 0).
- * A GSP-owned internalFlags variant used to sit between V1 and V2; it wedged
+ * Channel-alloc variants tried in table order after the TSG exists. V5
+ * (tsg-vas-pdb-skip-scrubber) is tried first: the pre-allocated
+ * page-directory-backed VAS is carried by the TSG (parent=TSG, channel
+ * hVASpace=0), with SKIP_SCRUBBER, PRIVILEGE_KERNEL and client memdescs. The
+ * fallbacks keep their historical numbers: V1 mirrors the RM's internal CE
+ * channel template (mem_utils_gm107.c:1186-1283: device parent, hDeviceId,
+ * SKIP_SCRUBBER flag, PRIVILEGE_KERNEL internalFlags); V3 keeps the reference
+ * shape but zeroes the memdescs and GPFIFO so the server allocates channel
+ * memory itself (no client ring to fill, so stage 4 refuses); V4 is the
+ * previously tried control shape (TSG parent, flags 0). A GSP-owned
+ * internalFlags variant (V2) used to sit between V1 and the rest; it wedged
  * the RM task and is removed so it can never be selected again.
  */
 struct gsp_phys_rmcall_channel_variant {
 	const char *name;
+	u32 id;			/* variant number used in logs and state */
 	bool tsg_parent;
 	u32 flags;
 	u32 internal_flags;
@@ -4711,15 +4772,19 @@ struct gsp_phys_rmcall_channel_variant {
 
 static const struct gsp_phys_rmcall_channel_variant
 gsp_phys_rmcall_channel_variants[] = {
-	{ "ref-device-skip-scrubber", false,
+	{ "tsg-vas-pdb-skip-scrubber", 5, true,
+	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
+	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
+	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 },
+	{ "ref-device-skip-scrubber", 1, false,
 	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
 	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
 	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 },
-	{ "server-allocated-mem", false,
+	{ "server-allocated-mem", 3, false,
 	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
 	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, true,
 	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 },
-	{ "current-control", true, 0,
+	{ "current-control", 4, true, 0,
 	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
 	  GSP_PHYS_RMCALL_HANDLE_CHANNEL },
 };
@@ -4746,8 +4811,11 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
 }
 
 /*
- * Stage 3: channel group (COPY0, physical, parent=device) then the channel.
- * The channel alloc walks the variants below in order (same already-held
+ * Stage 3: explicit VAS with its page directory, then channel group (COPY0,
+ * physical) and the channel. The VAS and PDB must both succeed: a channel
+ * whose VAS has no page-directory base fails the context-share/instance-block
+ * setup with NV_ERR_INVALID_STATE, and no variant avoids the VAS. The channel
+ * alloc then walks the variants below in order (same already-held
  * gsp_phys_rm_lock and gates); the first rpc_result==0 wins. Failed variants
  * log one compact line each and no payload dump, and an all-failed walk ends
  * in one summary line with every variant's rpc_result/status. Then
@@ -4759,12 +4827,62 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
  */
 static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 {
+	struct gsp_phys_rmcall_vas_params vas;
+	struct gsp_phys_rmcall_pdb_params pdb;
 	struct gsp_phys_rmcall_tsg_params tsg;
 	struct gsp_phys_rmcall_channel_params ch;
+	u32 vas_rpc = 0, vas_status = 0, pdb_rpc = 0;
 	u32 token = 0;
 	int ret;
 
+	/*
+	 * The TSG below carries this externally-owned FERMI_VASPACE_A, whose
+	 * root page directory is committed from the zeroed VASPDB chunk before
+	 * any channel alloc (the instance-block setup programs that base and
+	 * fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152). Either call
+	 * aborts the ladder: the variant walk cannot succeed without them.
+	 */
+	memset(&vas, 0, sizeof(vas));
+	vas.index = cpu_to_le32(GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW);
+	vas.flags = cpu_to_le32(GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED |
+				GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS);
+	vas.va_size = cpu_to_le64(GSP_PHYS_RMCALL_VAS_SIZE);
+	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
+				    boot_stage.rmcall_device,
+				    GSP_PHYS_RMCALL_HANDLE_VAS,
+				    GSP_PHYS_RMCALL_CLASS_VASPACE,
+				    &vas, sizeof(vas), false, NULL,
+				    "FERMI_VASPACE_A", true,
+				    &vas_rpc, &vas_status);
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: stage 3 aborted: FERMI_VASPACE_A alloc failed (rpc_result=0x%x status=0x%x); no VAS for the channel\n",
+		       vas_rpc, vas_status);
+		return ret;
+	}
+	boot_stage.rmcall_vas = GSP_PHYS_RMCALL_HANDLE_VAS;
+
+	memset(&pdb, 0, sizeof(pdb));
+	pdb.phys_address = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB], 0));
+	pdb.num_entries = cpu_to_le32(GSP_PHYS_RMCALL_PDB_NUM_ENTRIES);
+	pdb.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH |
+				GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS);
+	pdb.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
+	pdb.ch_id = 0;
+	pdb.sub_device_id = cpu_to_le32(GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID);
+	pdb.pasid = 0;
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
+				      GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY,
+				      &pdb, sizeof(pdb), 16, NULL, 0,
+				      &pdb_rpc, "DMA_SET_PAGE_DIRECTORY");
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: stage 3 aborted: DMA_SET_PAGE_DIRECTORY on hVas=0x%x failed (rpc_result=0x%x); the VAS has no page-directory base\n",
+		       boot_stage.rmcall_vas, pdb_rpc);
+		return ret;
+	}
+
 	memset(&tsg, 0, sizeof(tsg));
+	tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
 	tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
 	/*
 	 * The TSG's hParent must be the DEVICE, not the subdevice: the server
@@ -4805,17 +4923,17 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 					GSP_PHYS_RMCALL_CLASS_CHANNEL,
 					&ch, sizeof(ch), false, NULL, v->name,
 					false, &v_rpc[i], &v_status[i]);
-			pr_info("gsp-phys-boot: rm-call: stage 3 channel variant %u/%zu (%s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n",
-				i + 1,
+			pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n",
+				v->id, i + 1,
 				ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
 				v->name, parent, v->handle, v->flags,
 				v->internal_flags, ret, v_rpc[i], v_status[i]);
 			if (!ret) {
 				boot_stage.rmcall_channel = v->handle;
-				boot_stage.rmcall_channel_variant = i + 1;
+				boot_stage.rmcall_channel_variant = v->id;
 				boot_stage.rmcall_channel_server_mem = v->server_mem;
-				pr_info("gsp-phys-boot: rm-call: stage 3 channel variant %u (%s) WON: hChannel=0x%x\n",
-					i + 1, v->name, v->handle);
+				pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n",
+					v->id, v->name, v->handle);
 				break;
 			}
 		}
@@ -4829,7 +4947,8 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 				pos += scnprintf(summary + pos,
 						 sizeof(summary) - pos,
 						 "%sV%u %s rpc=0x%x status=0x%x",
-						 i ? "; " : "", i + 1,
+						 i ? "; " : "",
+						 gsp_phys_rmcall_channel_variants[i].id,
 						 gsp_phys_rmcall_channel_variants[i].name,
 						 v_rpc[i], v_status[i]);
 			pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n",
@@ -4842,7 +4961,8 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
 				      GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
 				      &token, sizeof(token), 28, &token,
-				      sizeof(token), "GET_WORK_SUBMIT_TOKEN");
+				      sizeof(token), NULL,
+				      "GET_WORK_SUBMIT_TOKEN");
 	if (ret) {
 		pr_warn("gsp-phys-boot: rm-call: work-submit token not available before GPFIFO_SCHEDULE (ret %d); stage 4 retries after scheduling\n",
 			ret);
@@ -4851,9 +4971,10 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 		boot_stage.rmcall_token = token;
 	}
 
-	pr_info("gsp-phys-boot: rm-call: stage 3 complete: hTsg=0x%x hChannel=0x%x variant=%u token=0x%x engine=COPY0 (cid is server-assigned, request value ignored)\n",
-		boot_stage.rmcall_tsg, boot_stage.rmcall_channel,
-		boot_stage.rmcall_channel_variant, boot_stage.rmcall_token);
+	pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x engine=COPY0 (cid is server-assigned, request value ignored)\n",
+		boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
+		boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant,
+		boot_stage.rmcall_token);
 	return 0;
 }
 
@@ -4941,7 +5062,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	 * submitted. Distinct error, nothing is scheduled or rung.
 	 */
 	if (boot_stage.rmcall_channel_server_mem) {
-		pr_err("gsp-phys-boot: rm-call: stage 4 unavailable: winning channel variant V%u (server-allocated mem) has no client GPFIFO/USERD; submission requires a client-memory variant (ref-device-skip-scrubber or current-control)\n",
+		pr_err("gsp-phys-boot: rm-call: stage 4 unavailable: winning channel variant V%u (server-allocated mem) has no client GPFIFO/USERD; submission requires a client-memory variant (V5 tsg-vas-pdb-skip-scrubber, V1 ref-device-skip-scrubber or V4 current-control)\n",
 		       boot_stage.rmcall_channel_variant);
 		return -EOPNOTSUPP;
 	}
@@ -4960,7 +5081,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
 				      GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE,
 				      sched_params, sizeof(sched_params), 16,
-				      NULL, 0, "GPFIFO_SCHEDULE");
+				      NULL, 0, NULL, "GPFIFO_SCHEDULE");
 	if (ret)
 		return ret;
 
@@ -4971,7 +5092,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 			bar0, boot_stage.rmcall_channel,
 			GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
 			&token, sizeof(token), 28, &token, sizeof(token),
-			"GET_WORK_SUBMIT_TOKEN(after schedule)");
+			NULL, "GET_WORK_SUBMIT_TOKEN(after schedule)");
 		if (ret) {
 			pr_err("gsp-phys-boot: rm-call: no work-submit token after GPFIFO_SCHEDULE; refusing to ring the doorbell\n");
 			return ret;
@@ -5290,12 +5411,13 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rm_probe_last_function,
 		   boot_stage.rm_probe_last_payload_len);
 	seq_printf(m,
-		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hTsg=0x%x hChannel=0x%x variant=%u token=0x%x\n",
+		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x\n",
 		   boot_stage.rmcall_ran, boot_stage.rmcall_stage,
 		   boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret,
 		   boot_stage.rmcall_rpcs, boot_stage.rmcall_client,
 		   boot_stage.rmcall_device, boot_stage.rmcall_subdevice,
-		   boot_stage.rmcall_tsg, boot_stage.rmcall_channel,
+		   boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
+		   boot_stage.rmcall_channel,
 		   boot_stage.rmcall_channel_variant,
 		   boot_stage.rmcall_token);
 	seq_printf(m,
@@ -6141,6 +6263,10 @@ static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) == 56,
 	      "rpc_alloc_memory_v13_01 first PTE must be at payload offset 56");
 static_assert(sizeof(struct gsp_phys_rmcall_mem_desc) == 24,
 	      "NV_MEMORY_DESC_PARAMS must be 24 bytes");
+static_assert(sizeof(struct gsp_phys_rmcall_vas_params) == 48,
+	      "NV_VASPACE_ALLOCATION_PARAMETERS must be 48 bytes (reference ABI)");
+static_assert(sizeof(struct gsp_phys_rmcall_pdb_params) == 32,
+	      "NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS must be 32 bytes");
 static_assert(sizeof(struct gsp_phys_rmcall_tsg_params) == 20,
 	      "NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS must be 20 bytes");
 static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368,
@@ -6171,8 +6297,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296
 	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
 	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
-static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 9,
-	      "the rm-call ladder registers exactly nine DMA buffers");
+static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 10,
+	      "the rm-call ladder registers exactly ten DMA buffers");
 static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
 	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
 	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,