gsp-phys: add the host RM-client ladder (gsp-phys-rm-call) for real object allocation and a first real copy

2 files changed, 1259 insertions, 8 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index c3620e5..59ac892 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -191,6 +191,154 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RPC_FUNC_SET_REGISTRY			73u
 
 /*
+ * Host RM client ladder (debugfs gsp-phys-rm-call). Wire ids verified against
+ * the 570.144 reference rpc_global_enums.h X-macro (the third argument is the
+ * wire value): ALLOC_MEMORY=4 (line 14), GSP_RM_CONTROL=76 (line 86),
+ * GSP_RM_ALLOC=103 (line 113). Classes/control commands:
+ *   NV01_ROOT 0x0 (class/cl0000.h:42), NV01_DEVICE_0 0x80 (class/cl0080.h),
+ *   NV20_SUBDEVICE_0 0x2080 (class/cl2080.h), KEPLER_CHANNEL_GROUP_A 0xA06C
+ *   (class/cla06c.h:33), TURING_CHANNEL_GPFIFO_A 0xC46F (class/clc46f.h:43),
+ *   TURING_DMA_COPY_A 0xC5B5 (class/clc5b5.h:33), NV01_MEMORY_LIST_SYSTEM
+ *   0x81 (class/cl84a0.h:45), NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN
+ *   0xC36F0108 (ctrl/ctrlc36f.h:82), NVA06F_CTRL_CMD_GPFIFO_SCHEDULE
+ *   0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), COPY0 engine type 0x9
+ *   (class/cl2080_notification.h:292).
+ */
+#define GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY	4u
+#define GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL	76u
+#define GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC	103u
+
+#define GSP_PHYS_RMCALL_CLASS_ROOT		0x0000u
+#define GSP_PHYS_RMCALL_CLASS_DEVICE		0x0080u
+#define GSP_PHYS_RMCALL_CLASS_SUBDEVICE		0x2080u
+#define GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM	0x0081u
+#define GSP_PHYS_RMCALL_CLASS_TSG		0xa06cu
+#define GSP_PHYS_RMCALL_CLASS_CHANNEL		0xc46fu
+#define GSP_PHYS_RMCALL_CLASS_CE		0xc5b5u
+#define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN	0xc36f0108u
+#define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE	0xa06f0103u
+#define GSP_PHYS_RMCALL_ENGINE_COPY0		0x9u
+
+/*
+ * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
+ * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
+ * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
+ * the server clears internalFlags and hPhysChannelGroup for GSP clients
+ * before it constructs the channel (gpu/fifo/kernel_channel.c:222-227), so
+ * the ladder does not rely on them for privilege - access control is the
+ * client's security context. They are kept so the request mirrors a
+ * kernel-client alloc on the CPU-RM path (kernel_channel.c:282-294).
+ */
+#define GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED	BIT(5)
+#define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL	0x2u
+
+/*
+ * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 (kernel/vgpu/
+ * rm_plugin_shared_code.h:66) and NV_MEMORY_UNCACHED=1 (kernel/os/
+ * nv_memory_type.h:31), which is the default RM system-memory CPU cache
+ * attribute (system_mem.c:73-74) and selects the NONCOHERENT_SYSMEM CE
+ * aperture (clc5b5.h:56-60).
+ */
+#define GSP_PHYS_RMCALL_ADDR_SYSMEM		1u
+#define GSP_PHYS_RMCALL_CACHE_UNCACHED		1u
+#define GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM	2u
+
+/* NVOS02 flags: PHYSICALITY field 7:4, NONCONTIGUOUS=1 -> 1 << 4 (nvos.h:189-191). */
+#define GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS	(1u << 4)
+
+/*
+ * Client-chosen handles for the objects the ladder creates itself. The server
+ * validates them with clientAssignResourceHandle (resserv/src/rs_client.c:
+ * 986-1008: nonzero, not the client handle, not a duplicate) inside the
+ * default RM resource handle range 0xcaf00000..0xcaf7ffff (rs_client.h:37-38).
+ */
+#define GSP_PHYS_RMCALL_HANDLE_DEVICE		0xcaf00000u
+#define GSP_PHYS_RMCALL_HANDLE_SUBDEVICE	0xcaf00004u
+#define GSP_PHYS_RMCALL_HANDLE_TSG		0xcaf00008u
+#define GSP_PHYS_RMCALL_HANDLE_CHANNEL		0xcaf0000cu
+#define GSP_PHYS_RMCALL_HANDLE_MEM_BASE		0xcaf01000u
+#define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE	4u
+
+/* Buffers the ladder allocates and registers in stage 2. */
+enum gsp_phys_rmcall_buf {
+	GSP_PHYS_RMCALL_BUF_SRC = 0,
+	GSP_PHYS_RMCALL_BUF_DST,
+	GSP_PHYS_RMCALL_BUF_SEM,
+	GSP_PHYS_RMCALL_BUF_PB,
+	GSP_PHYS_RMCALL_BUF_GPFIFO,
+	GSP_PHYS_RMCALL_BUF_USERD,
+	GSP_PHYS_RMCALL_BUF_INSTANCE,
+	GSP_PHYS_RMCALL_BUF_RAMFC,
+	GSP_PHYS_RMCALL_BUF_MTHDBUF,
+	GSP_PHYS_RMCALL_BUF_COUNT
+};
+
+#define GSP_PHYS_RMCALL_COPY_SIZE	0x1000u
+#define GSP_PHYS_RMCALL_GPFIFO_ENTRIES	8u
+#define GSP_PHYS_RMCALL_USERD_GPPUT_OFF	0x8cu	/* Nvc46fControl GPPut (clc46f.h:57) */
+#define GSP_PHYS_RMCALL_USERD_GPGET_OFF	0x88u	/* Nvc46fControl GPGet (clc46f.h:56) */
+#define GSP_PHYS_RMCALL_SEM_PAYLOAD	0xc0ffee01u
+#define GSP_PHYS_RMCALL_TIMEOUT_MS	2000u
+#define GSP_PHYS_RMCALL_POLL_MS		5u
+
+/*
+ * PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102
+ * writes GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_DOORBELL, token)
+ * (kernel/fifo/arch/turing/kernel_fifo_tu102.c:46-59); GPU_VREG_WR32 adds
+ * gpuGetVirtRegPhysOffset_TU102 (kern_gpu_tu102.c:93-100), which for a
+ * non-SRIOV PF is DRF_BASE(NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET) = 0xb80000
+ * (swref/published/turing/tu102/dev_vm.h:28); the register itself is
+ * NV_VIRTUAL_FUNCTION_DOORBELL at +0x30090 (dev_vm.h:229). The token the
+ * server returns from GET_WORK_SUBMIT_TOKEN already carries
+ * RUNLIST_ID[22:16] | VECTOR[11:0] (dev_ctrl.h:84-85, kernel_fifo_tu102.c:
+ * 118-121), so the host writes it unchanged.
+ */
+#define GSP_PHYS_RMCALL_DOORBELL_OFF	0xbb0090u
+
+/* MFREE subchannel method headers: OPCODE_IMM=1 31:29, COUNT 28:16,
+ * SUBCHANNEL 15:13, ADDRESS=(method >> 2) 11:0 (class/cl906f.h:185-189;
+ * generated/g_mem_mgr_nvoc.h:372-378); CE subchannel 4 (cla06fsubch.h:30). */
+#define GSP_PHYS_RMCALL_METHOD_IMM	1u
+#define GSP_PHYS_RMCALL_CE_SUBCHANNEL	4u
+
+/*
+ * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from
+ * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248,
+ * SET_SRC_PHYS_MODE 0x260, SET_DST_PHYS_MODE 0x264, LAUNCH_DMA 0x300,
+ * OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/LOWER 0x408/0x40c,
+ * PITCH_IN/OUT 0x410/0x414, LINE_LENGTH_IN/LINE_COUNT 0x418/0x41c.
+ * The method order and the physical-mode/pitch/launch encoding mirror
+ * _ceChannelPushMethodsBlock_GM107 (gpu/mem_mgr/arch/maxwell/
+ * mem_utils_gm107.c:1968-2118) and the semaphore payload block
+ * (gpu/mem_mgr/channel_utils.c:812-820).
+ */
+#define GSP_PHYS_RMCALL_CE_SET_OBJECT		0x000u
+#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A	0x240u
+#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B	0x244u
+#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD	0x248u
+#define GSP_PHYS_RMCALL_CE_SET_SRC_PHYS_MODE	0x260u
+#define GSP_PHYS_RMCALL_CE_SET_DST_PHYS_MODE	0x264u
+#define GSP_PHYS_RMCALL_CE_LAUNCH_DMA		0x300u
+#define GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER	0x400u
+#define GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER	0x404u
+#define GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER	0x408u
+#define GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER	0x40cu
+#define GSP_PHYS_RMCALL_CE_PITCH_IN		0x410u
+#define GSP_PHYS_RMCALL_CE_PITCH_OUT		0x414u
+#define GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN	0x418u
+#define GSP_PHYS_RMCALL_CE_LINE_COUNT		0x41cu
+
+/* LAUNCH_DMA: PIPELINED 1:0=1, FLUSH_ENABLE bit2, SEMAPHORE_TYPE 4:3=
+ * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE PHYSICAL
+ * bits12/13 (clc5b5.h:69-108); the physical memcopy sets exactly this shape
+ * (mem_utils_gm107.c:2082-2097), including FLUSH_ENABLE when the payload
+ * semaphore is requested. */
+#define GSP_PHYS_RMCALL_LAUNCH_DMA			\
+	(0x1u | (0x1u << 2) | (0x1u << 3) |		\
+	 (0x1u << 7) | (0x1u << 8) |			\
+	 (0x1u << 12) | (0x1u << 13))
+
+/*
  * PCIe config-space mirror window, as carried in GspSystemInfo. The field is
  * a BAR0 register offset, not a host buffer: the GSP-RM server feeds it to
  * osPciInitHandle()/osPciReadDword(), which do sysBar0Read(base + config
@@ -631,6 +779,25 @@ static struct gsp_phys_boot_stage {
 	u32 tx_seq_num;		/* next command-queue element seqNum */
 	u32 status_read_ptr;	/* next status-queue slot to consume (persists
 				 * across stage 3e and debugfs probe walks) */
+	/*
+	 * Host RM client ladder (debugfs gsp-phys-rm-call). Inert until the file
+	 * is written; stage persists across writes within one session.
+	 */
+	bool rmcall_ran;		/* the ladder ran at least once */
+	u32 rmcall_stage;		/* highest stage completed (0..4) */
+	u32 rmcall_last_stage;		/* stage the last run stopped in */
+	int rmcall_last_ret;		/* last run status (0 = success) */
+	u32 rmcall_rpcs;		/* RPCs submitted by the ladder */
+	u32 rmcall_client;		/* NV01_ROOT handle (server-chosen) */
+	u32 rmcall_device;
+	u32 rmcall_subdevice;
+	u32 rmcall_tsg;
+	u32 rmcall_channel;
+	u32 rmcall_token;		/* GET_WORK_SUBMIT_TOKEN result */
+	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
+	struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
+	u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
+	bool rmcall_mem_registered[GSP_PHYS_RMCALL_BUF_COUNT];
 	struct gsp_phys_boot_chunk fmc;
 	struct gsp_phys_radix3 radix3;
 	struct gsp_phys_boot_chunk signature;
@@ -1114,6 +1281,36 @@ static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where)
 		s.booter_binary_version, where);
 }
 
+/*
+ * Clear every piece of host-RM-client ladder state (stage progress, handles,
+ * token, registration flags and chunk descriptors). It does NOT free DMA
+ * chunks: gsp_phys_rmcall_fini() frees first, and gsp_phys_boot_prime() may
+ * call this only when no ladder allocation is live (primed=false means a
+ * previous staging_free() already freed and cleared everything).
+ */
+static void gsp_phys_rmcall_reset(void)
+{
+	u32 i;
+
+	boot_stage.rmcall_ran = false;
+	boot_stage.rmcall_stage = 0;
+	boot_stage.rmcall_last_stage = 0;
+	boot_stage.rmcall_last_ret = 0;
+	boot_stage.rmcall_rpcs = 0;
+	boot_stage.rmcall_client = 0;
+	boot_stage.rmcall_device = 0;
+	boot_stage.rmcall_subdevice = 0;
+	boot_stage.rmcall_tsg = 0;
+	boot_stage.rmcall_channel = 0;
+	boot_stage.rmcall_token = 0;
+	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
+		boot_stage.rmcall_mem_handles[i] = 0;
+		boot_stage.rmcall_mem_registered[i] = false;
+		memset(&boot_stage.rmcall_chunks[i], 0,
+		       sizeof(boot_stage.rmcall_chunks[i]));
+	}
+}
+
 int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev,
 			void __iomem *bar0,
 			const struct gsp_phys_boot_sources *src, bool dryrun)
@@ -1143,6 +1340,15 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev,
 		     GSP_PHYS_PAGE_SIZE;
 
 	boot_stage.dryrun = dryrun;
+	/*
+	 * Borrow the claim's pci_dev for the debugfs RM-client ladder (chunk
+	 * alloc/free). Valid for as long as the staging lives: gsp_phys_gpu.c
+	 * calls staging_free with the same phys.pdev while the claim is held.
+	 * (Re)initialise the ladder state here so a re-prime can never reuse
+	 * stale handles, tokens or registration flags from a previous session.
+	 */
+	boot_stage.pdev = pdev;
+	gsp_phys_rmcall_reset();
 	pr_info("gsp-phys-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n",
 		dryrun ? "prime" : "staging", pci_name(pdev),
 		src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE,
@@ -2458,16 +2664,23 @@ static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq,
 		if (ret)
 			return ret;
 
-		pr_info("gsp-phys-boot: rm-probe: status record slot %u (seqNum=%u elemCount=%u rpc.len=%u rpc.seq=%u checksum=0x%08x): function=0x%x rpc_result=%u\n",
-			one.first_slot, one.seq_num, one.elem_count,
-			one.rpc_length, one.rpc_sequence, one.checksum,
-			one.function, one.rpc_result);
-
+		/*
+		 * Matched records are logged by the caller, not here. Unmatched
+		 * async events are consumed (readPtr advances) and must not be
+		 * dropped silently: NOCAT journals feed the bounded unique
+		 * summary; everything else is debug-only (no unbounded logs).
+		 */
 		if (one.function == function &&
 		    (!match_sequence || one.rpc_sequence == sequence)) {
 			*rec = one;
 			return 0;
 		}
+		if (one.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD)
+			gsp_phys_nocat_record(&one);
+		else
+			pr_debug("gsp-phys-boot: status wait: dropping unmatched record slot %u (seqNum=%u rpc.len=%u checksum=0x%08x): function=0x%x rpc_result=%u\n",
+				 one.first_slot, one.seq_num, one.rpc_length,
+				 one.checksum, one.function, one.rpc_result);
 		gsp_phys_rx_record_free(&one);
 	}
 }
@@ -3785,6 +3998,854 @@ static int gsp_phys_rm_probe(void __iomem *bar0)
 }
 
 /* ============================================================
+ * Host RM client ladder (R1): gsp-phys-rm-call
+ * ============================================================ */
+
+/*
+ * Wire layouts, replicated from the 570.144 reference client:
+ *
+ * - rpc_gsp_rm_alloc_v03_00 (generated/g_rpc-structures.h:1373-1384): 32-byte
+ *   fixed prefix hClient/hParent/hObject/hClass/status/paramsSize/flags/
+ *   reserved[4], then params[]. The server writes the per-call status at
+ *   payload+16 and, for NV01_ROOT with hObject=0, the chosen client handle
+ *   into params[0] (rmapi/client.c:218-220), which is how
+ *   rmapiutilAllocClientAndDeviceHandles (rmapi/rmapi_utils.c:54-87) and this
+ *   ladder learn hClient. flags bit1 (RMAPI_RPC_FLAGS_SERIALIZED) means the
+ *   params are FINN-serialized; this ladder sends plain params, flags=0.
+ *
+ * - rpc_gsp_rm_control_v03_00 (generated/g_rpc-structures.h:1388-1397):
+ *   24-byte fixed prefix hClient/hObject/cmd/status/paramsSize/flags, then
+ *   params[]. The server writes the control status at payload+12 and echoes
+ *   params (rpc.c:9894-9998).
+ *
+ * - rpc_alloc_memory_v13_01 (generated/g_rpc-structures.h:79-91): fixed 44
+ *   bytes through pageCount plus struct pte_desc (4-byte idr/length header
+ *   padded to 8; sdk-structures.h:183-193) and the u64 PFN array. struct
+ *   pte_desc is 8-byte aligned, so pteDesc sits at payload offset 48 and the
+ *   first PTE at 56 (_issuePteDescRpc computes recordSize from exactly those
+ *   compile-time offsets, rpc.c:2040-2067). For IS_GSP_CLIENT the client
+ *   sends the real PTE count with PFN = physical address >> RM_PAGE_SHIFT
+ *   (rpc.c:3238-3247, 2064-2067).
+ */
+struct gsp_phys_rmcall_rm_alloc_req {
+	__le32 h_client;
+	__le32 h_parent;
+	__le32 h_object;
+	__le32 h_class;
+	__le32 status;
+	__le32 params_size;
+	__le32 flags;
+	u8 reserved[4];
+	u8 params[];
+};
+
+struct gsp_phys_rmcall_rm_control_req {
+	__le32 h_client;
+	__le32 h_object;
+	__le32 cmd;
+	__le32 status;
+	__le32 params_size;
+	__le32 flags;
+	u8 params[];
+};
+
+struct gsp_phys_rmcall_pte_desc {
+	__le32 idr_len;		/* idr:2 | reserved1:14 | length:16 */
+	__le32 pad;
+};
+
+struct gsp_phys_rmcall_alloc_memory_req {
+	__le32 h_client;
+	__le32 h_device;
+	__le32 h_memory;
+	__le32 h_class;
+	__le32 flags;
+	__le32 pte_adjust;
+	__le32 format;
+	__le32 pad0;
+	__le64 length;
+	__le32 page_count;
+	__le32 pad1;		/* struct pte_desc has 8-byte alignment */
+	struct gsp_phys_rmcall_pte_desc pte_desc;
+	__le64 pte[];
+};
+
+/* NV_MEMORY_DESC_PARAMS (alloc/alloc_channel.h:37-42). */
+struct gsp_phys_rmcall_mem_desc {
+	__le64 base;
+	__le64 size;
+	__le32 address_space;
+	__le32 cache_attrib;
+};
+
+/* NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS (nvos.h:2511-2518): hParent is the
+ * subdevice, engineType the group engine (COPY0), hVASpace 0 for a physical
+ * group. */
+struct gsp_phys_rmcall_tsg_params {
+	__le32 h_object_error;
+	__le32 h_object_ecc_error;
+	__le32 h_va_space;
+	__le32 engine_type;
+	u8 b_is_calling_context_vgpu_plugin;
+	u8 pad[3];
+};
+
+/*
+ * NV_CHANNEL_ALLOC_PARAMS (alloc/alloc_channel.h:296-342), 368 bytes.
+ * hUserdMemory[0]=0 with a physical userdMem descriptor is legal
+ * (alloc/alloc_channel.h:313-320). hParent is the channel group allocated in
+ * this stage, so the server attaches the channel to it
+ * (gpu/fifo/kernel_channel.c:359-436); TSG channels must pass hVASpace=0
+ * (kernel_channel.c:429-435).
+ */
+struct gsp_phys_rmcall_channel_params {
+	__le32 h_object_error;
+	__le32 h_object_buffer;
+	__le64 gp_fifo_offset;
+	__le32 gp_fifo_entries;
+	__le32 flags;
+	__le32 h_context_share;
+	__le32 h_va_space;
+	__le32 h_userd_memory[8];
+	__le64 userd_offset[8];
+	__le32 engine_type;
+	__le32 cid;
+	__le32 sub_device_id;
+	__le32 h_object_ecc_error;
+	struct gsp_phys_rmcall_mem_desc instance_mem;
+	struct gsp_phys_rmcall_mem_desc userd_mem;
+	struct gsp_phys_rmcall_mem_desc ramfc_mem;
+	struct gsp_phys_rmcall_mem_desc mthdbuf_mem;
+	__le32 h_phys_channel_group;
+	__le32 internal_flags;
+	struct gsp_phys_rmcall_mem_desc error_notifier_mem;
+	struct gsp_phys_rmcall_mem_desc ecc_error_notifier_mem;
+	__le32 process_id;
+	__le32 sub_process_id;
+	__le32 encrypt_iv[3];
+	__le32 decrypt_iv[3];
+	__le32 hmac_nonce[8];
+	__le32 tpc_config_id;
+};
+
+/* Bounded little-endian read of a reply payload word: a read past the
+ * received payload is never performed (off+4 <= payload_len). */
+static u32 gsp_phys_rmcall_reply_u32(const struct gsp_phys_rx_record *rec,
+				     u32 off)
+{
+	if (off + 4u > rec->payload_len)
+		return 0;
+	return get_unaligned_le32(rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
+				  off);
+}
+
+/* First 48 bytes of a reply payload at dynamic-debug level only (cheap). */
+static void gsp_phys_rmcall_dump_reply(const struct gsp_phys_rx_record *rec)
+{
+	u32 n;
+
+	if (!rec->buf || !rec->payload_len)
+		return;
+	n = min_t(u32, 48u, rec->payload_len);
+	print_hex_dump_debug("gsp-phys-boot: rm-call: reply: ",
+			     DUMP_PREFIX_OFFSET, 16, 4,
+			     rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, n,
+			     false);
+}
+
+/*
+ * One ladder RPC: submit, then wait up to GSP_PHYS_RMCALL_TIMEOUT_MS for the
+ * reply with the same function id. rec is always owned by the caller (freed
+ * with gsp_phys_rx_record_free(); zeroed when nothing arrived). Aborts on the
+ * first nonzero rpc_result or a short reply.
+ */
+static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function,
+			       const void *req, u32 req_len, u32 reply_min,
+			       struct gsp_phys_rx_record *rec)
+{
+	struct gsp_phys_status_queue sq;
+	u32 tx_seq = boot_stage.tx_seq_num;
+	int ret;
+
+	memset(rec, 0, sizeof(*rec));
+	ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len);
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: fn%u submit refused: %d\n",
+		       function, ret);
+		return ret;
+	}
+	boot_stage.rmcall_rpcs++;
+
+	gsp_phys_status_queue_init(&sq);
+	ret = gsp_phys_status_wait(&sq, function, false, 0,
+				   GSP_PHYS_RMCALL_TIMEOUT_MS, rec);
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: fn%u no reply within %u ms (%d)\n",
+		       function, GSP_PHYS_RMCALL_TIMEOUT_MS, ret);
+		return ret;
+	}
+
+	pr_info("gsp-phys-boot: rm-call: fn%u (txSeqNum=%u) reply: rpc.length=%u payload=%u seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n",
+		function, tx_seq, rec->rpc_length, rec->payload_len,
+		rec->seq_num, rec->rpc_sequence, rec->rpc_result,
+		rec->rpc_result_private, rec->checksum);
+	gsp_phys_rmcall_dump_reply(rec);
+
+	if (rec->rpc_result != 0) {
+		pr_err("gsp-phys-boot: rm-call: fn%u server rpc_result=0x%x (not success)\n",
+		       function, rec->rpc_result);
+		return -EIO;
+	}
+	if (rec->payload_len < reply_min) {
+		pr_err("gsp-phys-boot: rm-call: fn%u reply payload %u B shorter than the %u B this step needs\n",
+		       function, rec->payload_len, reply_min);
+		return -EIO;
+	}
+	return 0;
+}
+
+/*
+ * One GSP_RM_ALLOC (103). The reply echoes the fixed struct and params, so the
+ * server status at payload+16 and (for NV01_ROOT) the chosen client handle at
+ * params[0] (payload+32) are read back from it.
+ */
+static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client,
+				 u32 h_parent, u32 h_object, u32 h_class,
+				 const void *params, u32 params_len,
+				 bool object_is_out, u32 *out_object,
+				 const char *what)
+{
+	struct gsp_phys_rmcall_rm_alloc_req *req;
+	struct gsp_phys_rx_record rec;
+	u32 req_len = sizeof(*req) + params_len;
+	u32 status;
+	int ret;
+
+	req = kvzalloc(req_len, GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+	req->h_client = cpu_to_le32(h_client);
+	req->h_parent = cpu_to_le32(h_parent);
+	req->h_object = cpu_to_le32(h_object);
+	req->h_class = cpu_to_le32(h_class);
+	req->params_size = cpu_to_le32(params_len);
+	req->flags = cpu_to_le32(0);
+	if (params_len)
+		memcpy(req->params, params, params_len);
+
+	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
+				  req, req_len, 20, &rec);
+	kvfree(req);
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n",
+		       what, h_class, ret);
+		gsp_phys_rx_record_free(&rec);
+		return ret;
+	}
+
+	/*
+	 * Bound every reply read by payload_len: a root reply must carry the
+	 * full fixed prefix plus params[0] (36 B) before the chosen handle is
+	 * read; the rpc helper already guaranteed >= 20 B for status.
+	 */
+	if (object_is_out && rec.payload_len < 36) {
+		pr_err("gsp-phys-boot: rm-call: %s reply payload %u B does not carry the chosen handle\n",
+		       what, rec.payload_len);
+		gsp_phys_rx_record_free(&rec);
+		return -EIO;
+	}
+	status = gsp_phys_rmcall_reply_u32(&rec, 16);
+	if (object_is_out) {
+		*out_object = gsp_phys_rmcall_reply_u32(&rec, 32);
+		pr_info("gsp-phys-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n",
+			what, h_class, status, *out_object);
+	} else {
+		pr_info("gsp-phys-boot: rm-call: %s alloc: hParent=0x%x hObject=0x%x hClass=0x%x status=0x%x\n",
+			what, h_parent, h_object, h_class, status);
+	}
+	gsp_phys_rx_record_free(&rec);
+
+	if (status != 0) {
+		pr_err("gsp-phys-boot: rm-call: %s alloc server status=0x%x\n",
+		       what, status);
+		return -EIO;
+	}
+	return 0;
+}
+
+/* Register one stage-2 DMA chunk with ALLOC_MEMORY (4). */
+static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index)
+{
+	struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[index];
+	struct gsp_phys_rmcall_alloc_memory_req *req;
+	struct gsp_phys_rx_record rec;
+	u32 pages = c->pages;
+	size_t req_len = offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) +
+			 (size_t)pages * sizeof(__le64);
+	u32 handle = GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
+		     index * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE;
+	u32 i;
+	int ret;
+
+	req = kvzalloc(req_len, GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
+	req->h_device = cpu_to_le32(boot_stage.rmcall_device);
+	req->h_memory = cpu_to_le32(handle);
+	req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM);
+	req->flags = cpu_to_le32(c->page_dma ?
+				 GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS : 0);
+	req->pte_adjust = 0;
+	req->format = 0;	/* NV_MMU_PTE_KIND_PITCH = 0 for sysmem */
+	req->length = cpu_to_le64((u64)pages * GSP_PHYS_PAGE_SIZE);
+	req->page_count = cpu_to_le32(pages);
+	req->pte_desc.idr_len = cpu_to_le32((pages & 0xffffu) << 16);
+	for (i = 0; i < pages; i++)
+		req->pte[i] = cpu_to_le64(gsp_phys_chunk_page_dma(c, i) >> 12);
+
+	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY,
+				  req, (u32)req_len, 0, &rec);
+	kvfree(req);
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n",
+		       index, handle, pages,
+		       (u64)gsp_phys_chunk_page_dma(c, 0), ret);
+		gsp_phys_rx_record_free(&rec);
+		return ret;
+	}
+	gsp_phys_rx_record_free(&rec);
+
+	boot_stage.rmcall_mem_handles[index] = handle;
+	boot_stage.rmcall_mem_registered[index] = true;
+	pr_info("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u registered: hMemory=0x%x class=0x%x %u page(s) dma=0x%llx%s\n",
+		index, handle, GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM, pages,
+		(u64)gsp_phys_chunk_page_dma(c, 0),
+		c->page_dma ? " (per-page PTEs, noncontiguous flag)" :
+			      " (single PTE)");
+	return 0;
+}
+
+/*
+ * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at
+ * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031).
+ */
+static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
+				   const void *params, u32 params_len,
+				   u32 reply_min, void *out, u32 out_len,
+				   const char *what)
+{
+	struct gsp_phys_rmcall_rm_control_req *req;
+	struct gsp_phys_rx_record rec;
+	u32 req_len = sizeof(*req) + params_len;
+	u32 status;
+	int ret;
+
+	req = kvzalloc(req_len, GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
+	req->h_object = cpu_to_le32(h_object);
+	req->cmd = cpu_to_le32(cmd);
+	req->params_size = cpu_to_le32(params_len);
+	req->flags = cpu_to_le32(0);
+	if (params_len)
+		memcpy(req->params, params, params_len);
+
+	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL,
+				  req, req_len, reply_min, &rec);
+	kvfree(req);
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n",
+		       what, cmd, h_object, ret);
+		gsp_phys_rx_record_free(&rec);
+		return ret;
+	}
+
+	/* Bound the status read by the received payload (reply_min >= 16). */
+	if (rec.payload_len < 16) {
+		pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x reply payload %u B carries no status word\n",
+		       what, cmd, rec.payload_len);
+		gsp_phys_rx_record_free(&rec);
+		return -EIO;
+	}
+	status = gsp_phys_rmcall_reply_u32(&rec, 12);
+	if (out && out_len && rec.payload_len > 24) {
+		u32 n = min_t(u32, out_len, rec.payload_len - 24);
+
+		memcpy(out, rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 24, n);
+	}
+	gsp_phys_rx_record_free(&rec);
+
+	pr_info("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x, params %u B) status=0x%x\n",
+		what, cmd, h_object, params_len, status);
+	if (status != 0) {
+		pr_err("gsp-phys-boot: rm-call: %s server status=0x%x\n",
+		       what, status);
+		return -EIO;
+	}
+	return 0;
+}
+
+/*
+ * Stage 1: NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0, the reference
+ * client/device/subdevice sequence (rmapi/rmapi_utils.c:54-103). The root
+ * params are the 120-byte NV0000_ALLOC_PARAMETERS the client WAR sends
+ * (rpc.c:10074-10080); hClient must be zero so the server generates it and
+ * writes it back (rmapi/client.c:218-220). Device params: deviceId 0,
+ * hClientShare = hClient (rmapi_utils.c:75-77). Subdevice: subDeviceId 0.
+ */
+static int gsp_phys_rmcall_stage1(void __iomem *bar0)
+{
+	u8 params[120];
+	int ret;
+
+	memset(params, 0, sizeof(params));
+	ret = gsp_phys_rmcall_alloc(bar0, 0, 0, 0,
+				    GSP_PHYS_RMCALL_CLASS_ROOT,
+				    params, sizeof(params), true,
+				    &boot_stage.rmcall_client, "NV01_ROOT");
+	if (ret)
+		return ret;
+
+	memset(params, 0, sizeof(params));
+	put_unaligned_le32(0, params + 0);				/* deviceId */
+	put_unaligned_le32(boot_stage.rmcall_client, params + 4);	/* hClientShare */
+	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
+				    boot_stage.rmcall_client,
+				    GSP_PHYS_RMCALL_HANDLE_DEVICE,
+				    GSP_PHYS_RMCALL_CLASS_DEVICE,
+				    params, 56, false, NULL, "NV01_DEVICE_0");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_device = GSP_PHYS_RMCALL_HANDLE_DEVICE;
+
+	memset(params, 0, sizeof(params));
+	put_unaligned_le32(0, params + 0);				/* subDeviceId */
+	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
+				    boot_stage.rmcall_device,
+				    GSP_PHYS_RMCALL_HANDLE_SUBDEVICE,
+				    GSP_PHYS_RMCALL_CLASS_SUBDEVICE,
+				    params, 4, false, NULL, "NV20_SUBDEVICE_0");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_subdevice = GSP_PHYS_RMCALL_HANDLE_SUBDEVICE;
+
+	pr_info("gsp-phys-boot: rm-call: stage 1 complete: hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n",
+		boot_stage.rmcall_client, boot_stage.rmcall_device,
+		boot_stage.rmcall_subdevice);
+	return 0;
+}
+
+/* One 4 KB coherent chunk per ladder buffer; all freed by rm-call fini. */
+static int gsp_phys_rmcall_alloc_chunks(void)
+{
+	u32 i;
+
+	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
+		struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[i];
+
+		if (c->pages)
+			continue;
+		if (!boot_stage.pdev) {
+			pr_err("gsp-phys-boot: rm-call: no claimed pci_dev for chunk %u\n",
+			       i);
+			return -EINVAL;
+		}
+		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1)) {
+			pr_err("gsp-phys-boot: rm-call: buffer %u allocation failed\n",
+			       i);
+			return -ENOMEM;
+		}
+	}
+	return 0;
+}
+
+/*
+ * Stage 2: allocate and register every DMA buffer the ladder uses. Each
+ * buffer becomes one NV01_MEMORY_LIST_SYSTEM object whose PTE list is the
+ * real system PFNs (rpcAllocMemory_v13_01, rpc.c:3208-3250; the server side
+ * is memlistConstruct for ADDR_SYSMEM, mem_mgr/mem_list.c:48-330).
+ */
+static int gsp_phys_rmcall_stage2(void __iomem *bar0)
+{
+	struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
+	u32 i;
+	int ret;
+
+	ret = gsp_phys_rmcall_alloc_chunks();
+	if (ret)
+		return ret;
+
+	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++)
+		if (i != GSP_PHYS_RMCALL_BUF_SRC)
+			gsp_phys_chunk_zero(&boot_stage.rmcall_chunks[i]);
+	memset(gsp_phys_chunk_page_va(src, 0), 0xa5, GSP_PHYS_RMCALL_COPY_SIZE);
+
+	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
+		if (boot_stage.rmcall_mem_registered[i])
+			continue;
+		ret = gsp_phys_rmcall_register_mem(bar0, i);
+		if (ret) {
+			pr_err("gsp-phys-boot: rm-call: stage 2 aborted at buffer %u\n",
+			       i);
+			return ret;
+		}
+	}
+
+	pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf zeroed)\n",
+		GSP_PHYS_RMCALL_BUF_COUNT);
+	return 0;
+}
+
+static struct gsp_phys_rmcall_mem_desc
+gsp_phys_rmcall_mem_desc(u32 index)
+{
+	struct gsp_phys_rmcall_mem_desc d;
+
+	d.base = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
+			&boot_stage.rmcall_chunks[index], 0));
+	d.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE);
+	d.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM);
+	d.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED);
+	return d;
+}
+
+/*
+ * Stage 3: channel group (COPY0, physical) then the channel with the
+ * registered physical descriptors, then GET_WORK_SUBMIT_TOKEN. The token is
+ * requested here as specified; because the server's TU102 generator requires
+ * the channel to be on a runlist already (kernel_fifo_tu102.c:110-116, which
+ * GPFIFO_SCHEDULE establishes at kernel_channel.c:3244), a nonzero status is
+ * logged and retried after the schedule in stage 4 instead of aborting.
+ */
+static int gsp_phys_rmcall_stage3(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_tsg_params tsg;
+	struct gsp_phys_rmcall_channel_params ch;
+	u32 token = 0;
+	int ret;
+
+	memset(&tsg, 0, sizeof(tsg));
+	tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
+	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
+				    boot_stage.rmcall_subdevice,
+				    GSP_PHYS_RMCALL_HANDLE_TSG,
+				    GSP_PHYS_RMCALL_CLASS_TSG,
+				    &tsg, sizeof(tsg), false, NULL,
+				    "KEPLER_CHANNEL_GROUP_A");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG;
+
+	memset(&ch, 0, sizeof(ch));
+	ch.gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0));
+	ch.gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
+	ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED);
+	ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
+	ch.cid = cpu_to_le32(1);	/* client hint only: the server assigns
+					 * its own unique cid (kernel_channel.c:
+					 * 212-215) */
+	ch.sub_device_id = cpu_to_le32(0);
+	ch.internal_flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL);
+	ch.instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE);
+	ch.userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
+	ch.ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC);
+	ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF);
+	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
+				    boot_stage.rmcall_tsg,
+				    GSP_PHYS_RMCALL_HANDLE_CHANNEL,
+				    GSP_PHYS_RMCALL_CLASS_CHANNEL,
+				    &ch, sizeof(ch), false, NULL,
+				    "TURING_CHANNEL_GPFIFO_A");
+	if (ret)
+		return ret;
+	boot_stage.rmcall_channel = GSP_PHYS_RMCALL_HANDLE_CHANNEL;
+
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
+				      GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
+				      &token, sizeof(token), 28, &token,
+				      sizeof(token), "GET_WORK_SUBMIT_TOKEN");
+	if (ret) {
+		pr_warn("gsp-phys-boot: rm-call: work-submit token not available before GPFIFO_SCHEDULE (ret %d); stage 4 retries after scheduling\n",
+			ret);
+		boot_stage.rmcall_token = 0;
+	} else {
+		boot_stage.rmcall_token = token;
+	}
+
+	pr_info("gsp-phys-boot: rm-call: stage 3 complete: hTsg=0x%x hChannel=0x%x token=0x%x engine=COPY0 (cid is server-assigned, request value ignored) descriptors instance=0x%llx userd=0x%llx ramfc=0x%llx mthdbuf=0x%llx\n",
+		boot_stage.rmcall_tsg, boot_stage.rmcall_channel,
+		boot_stage.rmcall_token,
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE], 0),
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD], 0),
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC], 0),
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_MTHDBUF], 0));
+	return 0;
+}
+
+static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data)
+{
+	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+	       (1u << 16) |
+	       ((u32)GSP_PHYS_RMCALL_CE_SUBCHANNEL << 13) |
+	       (method >> 2);
+	*p++ = data;
+	return p;
+}
+
+/*
+ * CE method block for one 4 KB sysmem-to-sysmem pitch copy with a release
+ * semaphore, mirroring _ceChannelPushMethodsBlock_GM107
+ * (gpu/mem_mgr/arch/maxwell/mem_utils_gm107.c:1968-2118) with the semaphore
+ * payload block from channel_utils.c:812-820 and the physical aperture choice
+ * (system_mem.c:73-74 -> NONCOHERENT_SYSMEM). Returns the block size in
+ * dwords (<= GPFIFO entry LENGTH field, bits 30:10).
+ */
+static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma,
+					 u64 dst_dma)
+{
+	u32 *p = pb;
+
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_OBJECT,
+				 GSP_PHYS_RMCALL_CLASS_CE);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A,
+				 (u32)(sem_dma >> 32));
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B,
+				 (u32)sem_dma);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD,
+				 GSP_PHYS_RMCALL_SEM_PAYLOAD);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER,
+				 (u32)(dst_dma >> 32));
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER,
+				 (u32)dst_dma);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_OUT,
+				 GSP_PHYS_RMCALL_COPY_SIZE);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER,
+				 (u32)(src_dma >> 32));
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER,
+				 (u32)src_dma);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_IN,
+				 GSP_PHYS_RMCALL_COPY_SIZE);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN,
+				 GSP_PHYS_RMCALL_COPY_SIZE);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_COUNT, 1);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SRC_PHYS_MODE,
+				 GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_DST_PHYS_MODE,
+				 GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM);
+	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LAUNCH_DMA,
+				 GSP_PHYS_RMCALL_LAUNCH_DMA);
+	return (u32)(p - pb);
+}
+
+/*
+ * Stage 4: GPFIFO_SCHEDULE, (retry) GET_WORK_SUBMIT_TOKEN, then the first
+ * real work item: write the CE method block to the pushbuffer, publish the
+ * GPFIFO entry and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the
+ * PF channel doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem
+ * semaphore word. Afterwards dst must equal src byte for byte.
+ */
+static int gsp_phys_rmcall_stage4(void __iomem *bar0)
+{
+	struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
+	struct gsp_phys_boot_chunk *ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO];
+	struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
+	struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+	struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
+	struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
+	u8 sched_params[2] = { 1, 0 };	/* NVA06F bEnable=1, bSkipSubmit=0 */
+	resource_size_t bar0_len;
+	u32 *pb_va, *userd, *ring_va;
+	u32 words, gpput, gpget, slot, e0, e1, poll;
+	u64 pb_dma, sem_dma, src_dma, dst_dma;
+	unsigned long deadline;
+	int ret;
+
+	if (!boot_stage.pdev) {
+		pr_err("gsp-phys-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n");
+		return -EINVAL;
+	}
+	bar0_len = pci_resource_len(boot_stage.pdev, 0);
+	if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) {
+		pr_err("gsp-phys-boot: rm-call: BAR0 length 0x%llx does not cover the PF doorbell at 0x%x\n",
+		       (u64)bar0_len, GSP_PHYS_RMCALL_DOORBELL_OFF);
+		return -EIO;
+	}
+
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
+				      GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE,
+				      sched_params, sizeof(sched_params), 16,
+				      NULL, 0, "GPFIFO_SCHEDULE");
+	if (ret)
+		return ret;
+
+	if (!boot_stage.rmcall_token) {
+		u32 token = 0;
+
+		ret = gsp_phys_rmcall_control(
+			bar0, boot_stage.rmcall_channel,
+			GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
+			&token, sizeof(token), 28, &token, sizeof(token),
+			"GET_WORK_SUBMIT_TOKEN(after schedule)");
+		if (ret) {
+			pr_err("gsp-phys-boot: rm-call: no work-submit token after GPFIFO_SCHEDULE; refusing to ring the doorbell\n");
+			return ret;
+		}
+		boot_stage.rmcall_token = token;
+	}
+
+	pb_va = gsp_phys_chunk_page_va(pb, 0);
+	ring_va = gsp_phys_chunk_page_va(ring, 0);
+	userd = gsp_phys_chunk_page_va(userd_c, 0);
+	pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0);
+	sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0);
+	src_dma = (u64)gsp_phys_chunk_page_dma(src, 0);
+	dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0);
+
+	words = gsp_phys_rmcall_build_methods(pb_va, sem_dma, src_dma, dst_dma);
+
+	gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
+	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
+	if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) {
+		pr_err("gsp-phys-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n",
+		       gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
+		return -EIO;
+	}
+	slot = gpput;
+
+	/*
+	 * e0: GET = pushbuffer address >> 2 in bits 31:2, FETCH/NO_CONTEXT_
+	 * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the address high
+	 * dword and LENGTH 30:10 in dwords, exactly as the reference RM's own
+	 * submission does it (mem_utils_gm107.c:1861-1868: NvU64_HI32(get) in
+	 * GET_HI, NvU64_LO32(get) >> 2 in GET). The physical channel model
+	 * addresses the ring and pushbuffer by their system DMA addresses.
+	 */
+	e0 = (u32)(pb_dma >> 2) & 0x3fffffffu;
+	e1 = (u32)((pb_dma >> 32) & 0xffu) | (words << 10);
+	put_unaligned_le32(e0, (u8 *)ring_va + slot * 8);
+	put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4);
+	dma_wmb();
+	put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES,
+			   (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
+	wmb();
+	writel(boot_stage.rmcall_token, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF);
+
+	pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x doorbell@0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
+		pb_dma, words, slot, gpget, gpput, e0, e1,
+		boot_stage.rmcall_token, GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma,
+		GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS);
+
+	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS);
+	for (;;) {
+		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
+		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
+			break;
+		if (time_after_eq(jiffies, deadline)) {
+			gpget = get_unaligned_le32((u8 *)userd +
+						   GSP_PHYS_RMCALL_USERD_GPGET_OFF);
+			gpput = get_unaligned_le32((u8 *)userd +
+						   GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
+			pr_err("gsp-phys-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n",
+			       GSP_PHYS_RMCALL_TIMEOUT_MS, poll,
+			       GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput);
+			return -ETIMEDOUT;
+		}
+		msleep(GSP_PHYS_RMCALL_POLL_MS);
+	}
+
+	if (memcmp(gsp_phys_chunk_page_va(src, 0),
+		   gsp_phys_chunk_page_va(dst, 0),
+		   GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
+		pr_err("gsp-phys-boot: rm-call: semaphore released but the destination does not match the source after %u bytes\n",
+		       GSP_PHYS_RMCALL_COPY_SIZE);
+		return -EIO;
+	}
+
+	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
+	pr_info("gsp-phys-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n",
+		poll, src_dma, dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget,
+		get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
+		words, boot_stage.rmcall_token);
+	return 0;
+}
+
+/* Serialised by gsp_phys_rm_lock; stage state persists across writes. */
+static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target)
+{
+	u32 stage;
+	int ret = 0;
+
+	mutex_lock(&gsp_phys_rm_lock);
+	/*
+	 * Defensive re-check: release may have committed between the trigger's
+	 * gate and this lock (claim dropping, the unload RPC already sent, or
+	 * GSP-RM not init-done). Once any of these is true no RPC is issued.
+	 */
+	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
+	    boot_stage.rm_unload_submitted) {
+		pr_info("gsp-phys-boot: rm-call: no-op: claimed=%u init_done=%u unload_submitted=%u\n",
+			gsp_phys_gpu_claimed(), boot_stage.rm_init_done,
+			boot_stage.rm_unload_submitted);
+		mutex_unlock(&gsp_phys_rm_lock);
+		return -EINVAL;
+	}
+
+	for (stage = boot_stage.rmcall_stage + 1; stage <= target; stage++) {
+		boot_stage.rmcall_last_stage = stage;
+		switch (stage) {
+		case 1:
+			ret = gsp_phys_rmcall_stage1(bar0);
+			break;
+		case 2:
+			ret = gsp_phys_rmcall_stage2(bar0);
+			break;
+		case 3:
+			ret = gsp_phys_rmcall_stage3(bar0);
+			break;
+		case 4:
+			ret = gsp_phys_rmcall_stage4(bar0);
+			break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+		if (ret) {
+			pr_err("gsp-phys-boot: rm-call: ladder aborted in stage %u (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n",
+			       stage, ret, boot_stage.rmcall_rpcs);
+			break;
+		}
+		boot_stage.rmcall_stage = stage;
+		pr_info("gsp-phys-boot: rm-call: stage %u complete (%u RPCs so far)\n",
+			stage, boot_stage.rmcall_rpcs);
+	}
+	if (!ret && target <= boot_stage.rmcall_stage)
+		pr_info("gsp-phys-boot: rm-call: ladder at stage %u (target %u; %u RPCs)\n",
+			boot_stage.rmcall_stage, target, boot_stage.rmcall_rpcs);
+
+	boot_stage.rmcall_ran = true;
+	boot_stage.rmcall_last_ret = ret;
+	mutex_unlock(&gsp_phys_rm_lock);
+	return ret;
+}
+
+/*
+ * Free every ladder chunk and clear all ladder state, so a re-prime in the
+ * same module load starts from zero (no stale server handles/tokens/stage).
+ * Safe on a partially built ladder.
+ */
+static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
+{
+	u32 i;
+
+	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++)
+		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_chunks[i]);
+	gsp_phys_rmcall_reset();
+}
+
+/* ============================================================
  * Debugfs read-only state snapshot (no device writes)
  * ============================================================ */
 
@@ -3872,6 +4933,14 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rm_probe_last_function,
 		   boot_stage.rm_probe_last_payload_len);
 	seq_printf(m,
+		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hTsg=0x%x hChannel=0x%x token=0x%x\n",
+		   boot_stage.rmcall_ran, boot_stage.rmcall_stage,
+		   boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret,
+		   boot_stage.rmcall_rpcs, boot_stage.rmcall_client,
+		   boot_stage.rmcall_device, boot_stage.rmcall_subdevice,
+		   boot_stage.rmcall_tsg, boot_stage.rmcall_channel,
+		   boot_stage.rmcall_token);
+	seq_printf(m,
 		   "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n",
 		   wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0),
 		   wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0),
@@ -4159,6 +5228,88 @@ static const struct file_operations gsp_phys_rm_probe_fops = {
 	.llseek = no_llseek,
 };
 
+/*
+ * gsp-phys-rm-call (write-only, mode 0200): writing N (1..4) runs the host RM
+ * client ladder cumulatively through stage N and remembers the reached stage
+ * for the rest of the session (write 4 after 2 only runs 3 and 4). Every
+ * other value or state (parameter off, GPU unclaimed, stage 3d not run,
+ * init-done not observed, unload already submitted, BAR0 unmapped) is a
+ * logged no-op. The whole ladder is serialised with the unload/probe triggers
+ * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched.
+ */
+static ssize_t gsp_phys_rm_call_write(struct file *file,
+				      const char __user *ubuf, size_t len,
+				      loff_t *ppos)
+{
+	char buf[8];
+	unsigned long value;
+	void __iomem *bar0;
+	int ret;
+
+	if (len == 0 || len >= sizeof(buf))
+		return -EINVAL;
+	if (copy_from_user(buf, ubuf, len))
+		return -EFAULT;
+	buf[len] = '\0';
+
+	if (kstrtoul(strim(buf), 0, &value))
+		return -EINVAL;
+	if (value < 1 || value > 4) {
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N)\n");
+		return len;
+	}
+	if (!gsp_phys_boot_enabled()) {
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: gsp_phys_boot is not enabled\n");
+		return len;
+	}
+	if (!gsp_phys_gpu_claimed()) {
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: the physical GPU is not claimed\n");
+		return len;
+	}
+	if (!boot_stage.primed) {
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage-3d staging is not primed\n");
+		return len;
+	}
+	if (!boot_stage.ran) {
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n",
+			boot_stage.primed, boot_stage.kicked,
+			boot_stage.dryrun);
+		return len;
+	}
+	if (!boot_stage.rm_init_done) {
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n",
+			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
+			boot_stage.ran);
+		return len;
+	}
+	if (boot_stage.rm_unload_submitted) {
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: UNLOADING_GUEST_DRIVER was already submitted; GSP-RM is suspending\n");
+		return len;
+	}
+	if (!boot_stage.pdev) {
+		pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: no claimed pci_dev for chunk allocation\n");
+		return len;
+	}
+	bar0 = gsp_phys_gpu_bar0();
+	if (!bar0) {
+		pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: BAR0 is not mapped\n");
+		return len;
+	}
+
+	ret = gsp_phys_rmcall_run(bar0, (u32)value);
+	pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n",
+		value, ret, boot_stage.rmcall_stage,
+		boot_stage.rmcall_last_stage, boot_stage.rmcall_rpcs);
+	return len;
+}
+
+static const struct file_operations gsp_phys_rm_call_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = gsp_phys_rm_call_write,
+	.llseek = no_llseek,
+};
+
 int gsp_phys_boot_debugfs_init(void)
 {
 	struct dentry *file;
@@ -4213,7 +5364,21 @@ int gsp_phys_boot_debugfs_init(void)
 			err);
 		return (int)err;
 	}
-	pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only} ready\n");
+	file = debugfs_create_file("gsp-phys-rm-call", 0200,
+				   gsp_phys_debugfs_root, NULL,
+				   &gsp_phys_rm_call_fops);
+	if (IS_ERR_OR_NULL(file)) {
+		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;
+
+		/*
+		 * The RM-client ladder is optional: the other triggers and
+		 * the state snapshot stay registered.
+		 */
+		pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-call creation failed: %ld (other triggers and state snapshot stay available)\n",
+			err);
+		return (int)err;
+	}
+	pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only} ready\n");
 	return 0;
 }
 
@@ -4257,9 +5422,10 @@ void gsp_phys_boot_staging_free(struct pci_dev *pdev)
 	    boot_stage.fmc.pages || boot_stage.signature.pages ||
 	    boot_stage.queue.pages || boot_stage.radix3.table.pages ||
 	    boot_stage.login.pages || boot_stage.logrm.pages ||
-	    boot_stage.radix3.nr_chunks)
-		pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM)\n");
+	    boot_stage.radix3.nr_chunks || boot_stage.rmcall_chunks[0].pages)
+		pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM) and any rm-call ladder buffers\n");
 
+	gsp_phys_rmcall_fini(pdev);
 	gsp_phys_radix3_free(pdev, &boot_stage.radix3);
 	gsp_phys_chunk_free(pdev, &boot_stage.fmc);
 	gsp_phys_chunk_free(pdev, &boot_stage.signature);
@@ -4419,6 +5585,74 @@ static_assert(offsetof(struct gsp_phys_nocat_journal_record, diag_buffer) ==
 	      "NOCAT diagBuffer must live at payload offset 180 (reference ABI)");
 
 /*
+ * Host RM client ladder wire ABI: the rpc_gsp_rm_alloc_v03_00 prefix is
+ * 32 bytes with params[] at 32 and status at 16; the rpc_gsp_rm_control_v03_00
+ * prefix is 24 bytes with params[] at 24 and status at 12; the
+ * rpc_alloc_memory_v13_01 layout matches the generated header offsets
+ * (length 32, pageCount 40, pteDesc 48, first PTE 56).
+ */
+static_assert(sizeof(struct gsp_phys_rmcall_rm_alloc_req) == 32,
+	      "rpc_gsp_rm_alloc_v03_00 fixed prefix must be 32 bytes");
+static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, status) == 16,
+	      "rpc_gsp_rm_alloc status must be at payload offset 16");
+static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, params) == 32,
+	      "rpc_gsp_rm_alloc params must start at payload offset 32");
+static_assert(sizeof(struct gsp_phys_rmcall_rm_control_req) == 24,
+	      "rpc_gsp_rm_control_v03_00 fixed prefix must be 24 bytes");
+static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, status) == 12,
+	      "rpc_gsp_rm_control status must be at payload offset 12");
+static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, params) == 24,
+	      "rpc_gsp_rm_control params must start at payload offset 24");
+static_assert(sizeof(struct gsp_phys_rmcall_pte_desc) == 8,
+	      "struct pte_desc header must be 8 bytes (4 fields + 4 pad)");
+static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, length) == 32,
+	      "rpc_alloc_memory_v13_01.length must be at payload offset 32");
+static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, page_count) == 40,
+	      "rpc_alloc_memory_v13_01.pageCount must be at payload offset 40");
+static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte_desc) == 48,
+	      "rpc_alloc_memory_v13_01.pteDesc must be at payload offset 48 (8-byte aligned)");
+static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) == 56,
+	      "rpc_alloc_memory_v13_01 first PTE must be at payload offset 56");
+static_assert(sizeof(struct gsp_phys_rmcall_mem_desc) == 24,
+	      "NV_MEMORY_DESC_PARAMS must be 24 bytes");
+static_assert(sizeof(struct gsp_phys_rmcall_tsg_params) == 20,
+	      "NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS must be 20 bytes");
+static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368,
+	      "NV_CHANNEL_ALLOC_PARAMS must be 368 bytes (reference ABI)");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, gp_fifo_offset) == 8,
+	      "NV_CHANNEL_ALLOC_PARAMS.gpFifoOffset must be at offset 8");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_userd_memory) == 32,
+	      "NV_CHANNEL_ALLOC_PARAMS.hUserdMemory must be at offset 32");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_offset) == 64,
+	      "NV_CHANNEL_ALLOC_PARAMS.userdOffset must be at offset 64");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, engine_type) == 128,
+	      "NV_CHANNEL_ALLOC_PARAMS.engineType must be at offset 128");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, sub_device_id) == 136,
+	      "NV_CHANNEL_ALLOC_PARAMS.subDeviceId must be at offset 136");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, instance_mem) == 144,
+	      "NV_CHANNEL_ALLOC_PARAMS.instanceMem must be at offset 144");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_mem) == 168,
+	      "NV_CHANNEL_ALLOC_PARAMS.userdMem must be at offset 168");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ramfc_mem) == 192,
+	      "NV_CHANNEL_ALLOC_PARAMS.ramfcMem must be at offset 192");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, mthdbuf_mem) == 216,
+	      "NV_CHANNEL_ALLOC_PARAMS.mthdbufMem must be at offset 216");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_phys_channel_group) == 240,
+	      "NV_CHANNEL_ALLOC_PARAMS.hPhysChannelGroup must be at offset 240");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, internal_flags) == 244,
+	      "NV_CHANNEL_ALLOC_PARAMS.internalFlags must be at offset 244");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296,
+	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
+	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
+static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 9,
+	      "the rm-call ladder registers exactly nine DMA buffers");
+static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
+	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
+	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
+	      "rm-call memory handles must stay inside the RM client handle range");
+
+/*
  * GspSystemInfo wire ABI, pinned against the reference header compile
  * (offsetof/sizeof of kernel/gpu/gsp/gsp_static_config.h, 570.144) and the
  * tracked ABI checks (gsp_abi_check.c:324-375). Reference sender payload is
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 7ed1635..935777f 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -138,6 +138,23 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0);
  *     sends when init-done was not observed and never touches WPR2. The
  *     static-info reply is decoded into dmesg (internal handles, split-VA
  *     flag, BAR1/2 PDE bases, FB geometry, engine caps, SRIOV/board fields).
+ *   - "gsp-phys-rm-call" (write-only, mode 0200): writing N (1..4) runs the
+ *     host RM client ladder cumulatively through stage N on a live, init-done
+ *     claim and remembers the reached stage for the rest of the session:
+ *     stage 1 allocates NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0
+ *     (function 103); stage 2 allocates and registers the nine DMA buffers
+ *     (function 4, NV01_MEMORY_LIST_SYSTEM with real PFNs); stage 3 allocates
+ *     KEPLER_CHANNEL_GROUP_A (COPY0) and TURING_CHANNEL_GPFIFO_A with the
+ *     physical instance/userd/ramfc/mthdbuf descriptors then requests the
+ *     work-submit token (function 76); stage 4 schedules the channel, writes
+ *     the CE method block plus GPFIFO entry and USERD GP_PUT, rings the PF
+ *     channel doorbell and polls the sysmem semaphore for a real 4 KB
+ *     sysmem-to-sysmem copy (function 76 + the CE doorbell path). Serialised
+ *     with the other triggers by gsp_phys_rm_lock, per-RPC bounded
+ *     (2000 ms), aborts at the first failure with no object teardown (UNLOAD
+ *     + rmmod is the fail-safe), never touches WPR2, and is inert until the
+ *     file is written. Ladder buffers are freed by
+ *     gsp_phys_boot_staging_free() on release.
  *   - "gsp-phys-state" (read-only, mode 0444): seq_file snapshot of the
  *     claimed/booted state (falcon registers, scratch14, both queue TX
  *     headers, a bounded read-only status-ring walk, the stage flags and the