gsp-phys: FBMEM error notifier and the bind/notif-index/token order before submission

1 files changed, 397 insertions, 112 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 920ccdb..3ba8714 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -199,10 +199,15 @@ bool gsp_phys_boot_requested(void)
  *   NV20_SUBDEVICE_0 0x2080 (class/cl2080.h), KEPLER_CHANNEL_GROUP_A 0xA06C
  *   (class/cla06c.h:33), TURING_CHANNEL_GPFIFO_A 0xC46F (class/clc46f.h:43),
  *   TURING_DMA_COPY_A 0xC5B5 (class/clc5b5.h:33), NV01_MEMORY_LIST_SYSTEM
- *   0x81 (class/cl84a0.h:45), NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN
- *   0xC36F0108 (ctrl/ctrlc36f.h:82), NVA06F_CTRL_CMD_GPFIFO_SCHEDULE
- *   0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), FERMI_VASPACE_A 0x90F1
- *   (class/cl90f1.h:30), NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY 0x801813
+ *   0x81 (class/cl84a0.h:45), NV01_MEMORY_LIST_FBMEM 0x82
+ *   (class/cl84a0.h:47), NV01_MEMORY_LOCAL_USER 0x40 (class/cl0040.h:34),
+ *   NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN 0xC36F0108
+ *   (ctrl/ctrlc36f.h:82), NVC36F_CTRL_CMD_GPFIFO_SET_WORK_SUBMIT_TOKEN_
+ *   NOTIF_INDEX 0xC36F010A (ctrl/ctrlc36f.h:135), NVA06F_CTRL_CMD_
+ *   GPFIFO_SCHEDULE 0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54),
+ *   NVA06F_CTRL_CMD_BIND 0xA06F0104 (ctrl/ctrla06f/ctrla06fgpfifo.h:83),
+ *   FERMI_VASPACE_A 0x90F1 (class/cl90f1.h:30),
+ *   NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY 0x801813
  *   (ctrl/ctrl0080dma.h:832-840), COPY0 engine type 0x9
  *   (class/cl2080_notification.h:292).
  */
@@ -214,16 +219,36 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_CLASS_DEVICE		0x0080u
 #define GSP_PHYS_RMCALL_CLASS_SUBDEVICE		0x2080u
 #define GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM	0x0081u
+#define GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER	0x0040u
 #define GSP_PHYS_RMCALL_CLASS_TSG		0xa06cu
 #define GSP_PHYS_RMCALL_CLASS_CHANNEL		0xc46fu
 #define GSP_PHYS_RMCALL_CLASS_CE		0xc5b5u
 #define GSP_PHYS_RMCALL_CLASS_VASPACE		0x90f1u
 #define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN	0xc36f0108u
+#define GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX	0xc36f010au
 #define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE	0xa06f0103u
+#define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL	0xa06f0104u
 #define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY	0x801813u
 #define GSP_PHYS_RMCALL_ENGINE_COPY0		0x9u
 
 /*
+ * Work-submit token notifier slot. The default index is
+ * NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN = 1
+ * (nvos.h:2465-2468); the reference ladder requests __SIZE_1 + deviceIndex
+ * = 3 for subdevice 0 before asking for the token
+ * (nvidia-push-init.c:303-316), so the 4 KB FBMEM notifier below must span at
+ * least (3 + 1) * sizeof(NvNotification) = 64 B.
+ */
+#define GSP_PHYS_RMCALL_NOTIF_INDEX		3u
+
+/* NVOS32 type of the error-notifier allocation (nvos.h:939); attr bits
+ * 26:25 = VIDMEM location and 31:29 = UNCACHED coherency are both zero, the
+ * shape the RM's own channel-notifier allocation uses
+ * (mem_utils_gm107.c:344-352). */
+#define GSP_PHYS_RMCALL_OS32_TYPE_IMAGE		0u
+#define GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED	0u
+
+/*
  * FERMI_VASPACE_A allocation (nvos.h:2756-2765): flags bit 3
  * (IS_EXTERNALLY_OWNED) and bit 8 (ALLOW_ZERO_ADDRESS); the 256 MB window is
  * the smallest deterministic size this ladder needs and keeps vaBase 0. The
@@ -262,13 +287,14 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL	0x2u
 
 /*
- * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 (kernel/vgpu/
- * rm_plugin_shared_code.h:66) and NV_MEMORY_UNCACHED=1 (kernel/os/
- * nv_memory_type.h:31), which is the default RM system-memory CPU cache
- * attribute (system_mem.c:73-74) and selects the NONCOHERENT_SYSMEM CE
+ * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 and ADDR_FBMEM=2
+ * (kernel/vgpu/rm_plugin_shared_code.h:66-67) and NV_MEMORY_UNCACHED=1
+ * (kernel/os/nv_memory_type.h:31), which is the default RM system-memory CPU
+ * cache attribute (system_mem.c:73-74) and selects the NONCOHERENT_SYSMEM CE
  * aperture (clc5b5.h:56-60).
  */
 #define GSP_PHYS_RMCALL_ADDR_SYSMEM		1u
+#define GSP_PHYS_RMCALL_ADDR_FBMEM		2u
 #define GSP_PHYS_RMCALL_CACHE_UNCACHED		1u
 #define GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM	2u
 
@@ -291,6 +317,7 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2	0xcaf00014u
 #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3	0xcaf00018u
 #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5	0xcaf00024u
+#define GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM	0xcaf00028u
 #define GSP_PHYS_RMCALL_HANDLE_MEM_BASE		0xcaf01000u
 #define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE	4u
 
@@ -327,6 +354,7 @@ enum gsp_phys_rmcall_buf {
 };
 
 #define GSP_PHYS_RMCALL_COPY_SIZE	0x1000u
+#define GSP_PHYS_RMCALL_NOTIFIER_SIZE	0x1000u
 #define GSP_PHYS_RMCALL_GPFIFO_ENTRIES	8u
 #define GSP_PHYS_RMCALL_USERD_GPPUT_OFF	0x8cu	/* Nvc46fControl GPPut (clc46f.h:57) */
 #define GSP_PHYS_RMCALL_USERD_GPGET_OFF	0x88u	/* Nvc46fControl GPGet (clc46f.h:56) */
@@ -861,6 +889,9 @@ static struct gsp_phys_boot_stage {
 					 * the variant table for its number) */
 	bool rmcall_channel_server_mem;	/* winning variant had server-allocated mem */
 	u32 rmcall_token;		/* GET_WORK_SUBMIT_TOKEN result */
+	u32 rmcall_notifier_mem;	/* FBMEM error-notifier object handle */
+	u64 rmcall_notifier_base;	/* its returned frame-buffer address */
+	bool rmcall_notifier_valid;	/* the FBMEM notifier alloc succeeded */
 	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
 	struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
 	u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
@@ -1373,6 +1404,9 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_channel_variant = 0;
 	boot_stage.rmcall_channel_server_mem = false;
 	boot_stage.rmcall_token = 0;
+	boot_stage.rmcall_notifier_mem = 0;
+	boot_stage.rmcall_notifier_base = 0;
+	boot_stage.rmcall_notifier_valid = false;
 	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
 		boot_stage.rmcall_mem_handles[i] = 0;
 		boot_stage.rmcall_mem_registered[i] = false;
@@ -4208,6 +4242,41 @@ struct gsp_phys_rmcall_mem_desc {
 	__le32 cache_attrib;
 };
 
+/*
+ * NV_MEMORY_ALLOCATION_PARAMS (nvos.h:1634-1668), 128 bytes. The u64 fields
+ * are 8-byte aligned; offset and limit are [IN/OUT], so a frame-buffer
+ * allocation returns the allocated FB offset (and its limit) in the echoed
+ * params. type/attr carry the NVOS32 enums (type IMAGE, location VIDMEM,
+ * coherency UNCACHED are all the zero/default values).
+ */
+struct gsp_phys_rmcall_memory_alloc_params {
+	__le32 owner;
+	__le32 type;
+	__le32 flags;
+	__le32 width;
+	__le32 height;
+	__le32 pitch;
+	__le32 attr;
+	__le32 attr2;
+	__le32 format;
+	__le32 compr_covg;
+	__le32 zcull_covg;
+	__le32 pad0;
+	__le64 range_lo;
+	__le64 range_hi;
+	__le64 size;
+	__le64 alignment;
+	__le64 offset;
+	__le64 limit;
+	__le64 address;
+	__le32 ctag_offset;
+	__le32 h_va_space;
+	__le32 internalflags;
+	__le32 tag;
+	__le32 numa_node;
+	__le32 pad1;
+};
+
 /* NV_VASPACE_ALLOCATION_PARAMETERS (nvos.h:2756-2765), 48 bytes: index, flags,
  * vaSize, vaStartInternal, vaLimitInternal, bigPageSize, vaBase. */
 struct gsp_phys_rmcall_vas_params {
@@ -4565,13 +4634,103 @@ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index)
 }
 
 /*
+ * Allocate the channel error-context notifier in FBMEM. On GSP the channel
+ * notify path rejects a non-FBMEM error-context memdesc with
+ * NV_ERR_INVALID_STATE (kernel_channel.c:1924-1925), and the server's channel
+ * constructor materializes that memdesc from NV_CHANNEL_ALLOC_PARAMS'
+ * errorNotifierMem base/size/addressSpace; the notifier slot it writes is
+ * notifyIndex (set to GSP_PHYS_RMCALL_NOTIF_INDEX by the stage-4 sequence).
+ *
+ * The memlist registration used for the sysmem ladder buffers cannot produce
+ * a safe FBMEM memdesc for a host client (classes 0x81/0x82 only reference
+ * frame-buffer pages a guest already owns; 0x82 would accept our host PFNs as
+ * literal FB offsets). The frame-buffer page is therefore allocated from the
+ * GSP-RM itself with GSP_RM_ALLOC (103) class NV01_MEMORY_LOCAL_USER (0x40),
+ * mirroring the RM's own channel-notifier allocation (type IMAGE, location
+ * VIDMEM, coherency UNCACHED; mem_utils_gm107.c:344-352); the server returns
+ * the allocated FB offset in NV_MEMORY_ALLOCATION_PARAMS.offset, which is what
+ * errorNotifierMem.base carries.
+ *
+ * Failure is non-fatal: the channel alloc then keeps a zeroed notifier
+ * memdesc (server SYSMEM/ADDR_UNKNOWN, the cycle-19 shape) and the stage-4
+ * GET_WORK_SUBMIT_TOKEN recovery path still obtains the token from the
+ * control reply. The FB object is a server object like the VAS/TSG/channel
+ * and is not torn down by rm-call fini (UNLOAD + rmmod remains the fail-safe).
+ */
+static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_memory_alloc_params mem;
+	struct gsp_phys_rmcall_rm_alloc_req *req;
+	struct gsp_phys_rx_record rec;
+	u32 req_len = sizeof(*req) + sizeof(mem);
+	u32 status;
+	u64 base;
+	int ret;
+
+	memset(&mem, 0, sizeof(mem));
+	mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE);
+	mem.size = cpu_to_le64(GSP_PHYS_RMCALL_NOTIFIER_SIZE);
+	mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED);
+
+	req = kvzalloc(req_len, GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
+	req->h_parent = cpu_to_le32(boot_stage.rmcall_device);
+	req->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM);
+	req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER);
+	req->params_size = cpu_to_le32(sizeof(mem));
+	memcpy(req->params, &mem, sizeof(mem));
+
+	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
+				  req, req_len, 20, &rec, true, NULL);
+	kvfree(req);
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: FBMEM error-notifier alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n",
+		       GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM,
+		       GSP_PHYS_RMCALL_NOTIFIER_SIZE, ret);
+		gsp_phys_rx_record_free(&rec);
+		return ret;
+	}
+
+	status = gsp_phys_rmcall_reply_u32(&rec, 16);
+	if (status != 0 ||
+	    rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params,
+					    offset) + 8) {
+		pr_err("gsp-phys-boot: rm-call: FBMEM error-notifier alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n",
+		       status, rec.payload_len,
+		       offsetof(struct gsp_phys_rmcall_memory_alloc_params,
+				offset) + 8);
+		gsp_phys_rx_record_free(&rec);
+		return -EIO;
+	}
+	base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 +
+				  offsetof(struct gsp_phys_rmcall_memory_alloc_params,
+					   offset));
+	gsp_phys_rx_record_free(&rec);
+
+	boot_stage.rmcall_notifier_mem = GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM;
+	boot_stage.rmcall_notifier_base = base;
+	boot_stage.rmcall_notifier_valid = true;
+	pr_info("gsp-phys-boot: rm-call: FBMEM error-notifier allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx (notifyIndex=%u)\n",
+		boot_stage.rmcall_notifier_mem, GSP_PHYS_RMCALL_NOTIFIER_SIZE,
+		boot_stage.rmcall_notifier_base, GSP_PHYS_RMCALL_NOTIF_INDEX);
+	return 0;
+}
+
+/*
  * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at
- * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031).
+ * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional)
+ * receives the server control status even when it is nonzero; the echoed
+ * params are copied into out before that status is checked, so a caller can
+ * still recover server-written output (the work-submit token) from a failed
+ * reply.
  */
 static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
 				   const void *params, u32 params_len,
 				   u32 reply_min, void *out, u32 out_len,
-				   u32 *out_rpc_result, const char *what)
+				   u32 *out_rpc_result, u32 *out_status,
+				   const char *what)
 {
 	struct gsp_phys_rmcall_rm_control_req *req;
 	struct gsp_phys_rx_record rec;
@@ -4579,6 +4738,8 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
 	u32 status;
 	int ret;
 
+	if (out_status)
+		*out_status = 0;
 	req = kvzalloc(req_len, GFP_KERNEL);
 	if (!req)
 		return -ENOMEM;
@@ -4609,6 +4770,8 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
 		return -EIO;
 	}
 	status = gsp_phys_rmcall_reply_u32(&rec, 12);
+	if (out_status)
+		*out_status = status;
 	if (out && out_len && rec.payload_len > 24) {
 		u32 n = min_t(u32, out_len, rec.payload_len - 24);
 
@@ -4806,6 +4969,22 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
 	ch->cid = cpu_to_le32(1);	/* client hint; server assigns its own */
 	ch->sub_device_id = cpu_to_le32(0);
 	ch->internal_flags = cpu_to_le32(v->internal_flags);
+	/*
+	 * The error-context notifier must be an FBMEM memdesc on GSP
+	 * (kernel_channel.c:1924-1925) and the server builds pErrContextMemDesc
+	 * from these fields; carry the stage-3 FBMEM allocation when it exists.
+	 * The other memdescs stay the client sysmem chunks.
+	 */
+	if (boot_stage.rmcall_notifier_valid) {
+		ch->error_notifier_mem.base =
+			cpu_to_le64(boot_stage.rmcall_notifier_base);
+		ch->error_notifier_mem.size =
+			cpu_to_le64(GSP_PHYS_RMCALL_NOTIFIER_SIZE);
+		ch->error_notifier_mem.address_space =
+			cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM);
+		ch->error_notifier_mem.cache_attrib =
+			cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED);
+	}
 	if (v->server_mem)
 		return;	/* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */
 	ch->gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
@@ -4818,28 +4997,94 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
 }
 
 /*
+ * Channel-alloc variant walk (same already-held gsp_phys_rm_lock and gates):
+ * the first rpc_result==0 wins and records its handle/variant/server-mem
+ * state. Failed variants log one compact line each and no payload dump; an
+ * all-failed walk ends in one summary line with every variant's
+ * rpc_result/status. Kept out of stage 3 so the inline stack stays small.
+ */
+static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_channel_params ch;
+	u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
+	u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
+	u32 i;
+	int ret;
+
+	ret = -EIO;
+	for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) {
+		const struct gsp_phys_rmcall_channel_variant *v =
+			&gsp_phys_rmcall_channel_variants[i];
+		u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg :
+					     boot_stage.rmcall_device;
+
+		gsp_phys_rmcall_channel_params_init(&ch, v);
+		v_rpc[i] = 0;
+		v_status[i] = 0;
+		ret = gsp_phys_rmcall_alloc(bar0,
+				boot_stage.rmcall_client, parent,
+				v->handle,
+				GSP_PHYS_RMCALL_CLASS_CHANNEL,
+				&ch, sizeof(ch), false, NULL, v->name,
+				false, &v_rpc[i], &v_status[i]);
+		pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n",
+			v->id, i + 1,
+			ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
+			v->name, parent, v->handle, v->flags,
+			v->internal_flags, ret, v_rpc[i], v_status[i]);
+		if (!ret) {
+			boot_stage.rmcall_channel = v->handle;
+			boot_stage.rmcall_channel_variant = v->id;
+			boot_stage.rmcall_channel_server_mem = v->server_mem;
+			pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n",
+				v->id, v->name, v->handle);
+			return 0;
+		}
+	}
+
+	{
+		char summary[256];
+		u32 pos = 0;
+
+		for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++)
+			pos += scnprintf(summary + pos, sizeof(summary) - pos,
+					 "%sV%u %s rpc=0x%x status=0x%x",
+					 i ? "; " : "",
+					 gsp_phys_rmcall_channel_variants[i].id,
+					 gsp_phys_rmcall_channel_variants[i].name,
+					 v_rpc[i], v_status[i]);
+		pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n",
+		       ARRAY_SIZE(gsp_phys_rmcall_channel_variants), summary);
+	}
+	return ret;
+}
+
+/*
  * Stage 3: explicit VAS with its page directory, then channel group (COPY0,
  * physical) and the channel. The VAS and PDB must both succeed: a channel
  * whose VAS has no page-directory base fails the context-share/instance-block
  * setup with NV_ERR_INVALID_STATE, and no variant avoids the VAS. The channel
- * alloc then walks the variants below in order (same already-held
- * gsp_phys_rm_lock and gates); the first rpc_result==0 wins. Failed variants
- * log one compact line each and no payload dump, and an all-failed walk ends
- * in one summary line with every variant's rpc_result/status. Then
- * GET_WORK_SUBMIT_TOKEN. The token is requested before scheduling as
- * specified; because the server's TU102 generator requires the channel to be
- * on a runlist already (kernel_fifo_tu102.c:110-116, which GPFIFO_SCHEDULE
- * establishes at kernel_channel.c:3244), a nonzero status is logged and
- * retried after the schedule in stage 4 instead of aborting.
+ * alloc then walks the variants in gsp_phys_rmcall_alloc_channel in order
+ * (same already-held gsp_phys_rm_lock and gates); the first rpc_result==0
+ * wins. Failed variants log one compact line each and no payload dump, and an
+ * all-failed walk ends in one summary line with every variant's
+ * rpc_result/status.
+ *
+ * Before the walk, one 4 KB FBMEM error-notifier page is allocated from the
+ * GSP-RM and its memdesc fields are carried in every channel variant (the
+ * server's GSP-only notify path requires the error-context notifier to live
+ * in ADDR_FBMEM, kernel_channel.c:1924-1925). The work-submit token is not
+ * requested here: its generator requires BIND + GPFIFO_SCHEDULE (runlist
+ * membership, kernel_fifo_tu102.c:110-116), so the whole BIND -> SCHEDULE ->
+ * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN order lives in
+ * stage 4.
  */
 static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_vas_params vas;
 	struct gsp_phys_rmcall_pdb_params pdb;
 	struct gsp_phys_rmcall_tsg_params tsg;
-	struct gsp_phys_rmcall_channel_params ch;
 	u32 vas_rpc = 0, vas_status = 0, pdb_rpc = 0;
-	u32 token = 0;
 	int ret;
 
 	/*
@@ -4881,7 +5126,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
 				      GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY,
 				      &pdb, sizeof(pdb), 16, NULL, 0,
-				      &pdb_rpc, "DMA_SET_PAGE_DIRECTORY");
+				      &pdb_rpc, NULL, "DMA_SET_PAGE_DIRECTORY");
 	if (ret) {
 		pr_err("gsp-phys-boot: rm-call: stage 3 aborted: DMA_SET_PAGE_DIRECTORY on hVas=0x%x numEntries=%u failed (rpc_result=0x%x); the VAS has no page-directory base\n",
 		       boot_stage.rmcall_vas, le32_to_cpu(pdb.num_entries),
@@ -4910,79 +5155,27 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 		return ret;
 	boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG;
 
-	{
-		u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
-		u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
-		u32 i;
-
-		ret = -EIO;
-		for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) {
-			const struct gsp_phys_rmcall_channel_variant *v =
-				&gsp_phys_rmcall_channel_variants[i];
-			u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg :
-						     boot_stage.rmcall_device;
-
-			gsp_phys_rmcall_channel_params_init(&ch, v);
-			v_rpc[i] = 0;
-			v_status[i] = 0;
-			ret = gsp_phys_rmcall_alloc(bar0,
-					boot_stage.rmcall_client, parent,
-					v->handle,
-					GSP_PHYS_RMCALL_CLASS_CHANNEL,
-					&ch, sizeof(ch), false, NULL, v->name,
-					false, &v_rpc[i], &v_status[i]);
-			pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n",
-				v->id, i + 1,
-				ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
-				v->name, parent, v->handle, v->flags,
-				v->internal_flags, ret, v_rpc[i], v_status[i]);
-			if (!ret) {
-				boot_stage.rmcall_channel = v->handle;
-				boot_stage.rmcall_channel_variant = v->id;
-				boot_stage.rmcall_channel_server_mem = v->server_mem;
-				pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n",
-					v->id, v->name, v->handle);
-				break;
-			}
-		}
-		if (ret) {
-			char summary[256];
-			u32 pos = 0;
-
-			for (i = 0;
-			     i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants);
-			     i++)
-				pos += scnprintf(summary + pos,
-						 sizeof(summary) - pos,
-						 "%sV%u %s rpc=0x%x status=0x%x",
-						 i ? "; " : "",
-						 gsp_phys_rmcall_channel_variants[i].id,
-						 gsp_phys_rmcall_channel_variants[i].name,
-						 v_rpc[i], v_status[i]);
-			pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n",
-			       ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
-			       summary);
-			return ret;
-		}
-	}
-
-	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
-				      GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
-				      &token, sizeof(token), 28, &token,
-				      sizeof(token), NULL,
-				      "GET_WORK_SUBMIT_TOKEN");
-	if (ret) {
-		pr_warn("gsp-phys-boot: rm-call: work-submit token not available before GPFIFO_SCHEDULE (ret %d); stage 4 retries after scheduling\n",
+	/*
+	 * Allocate the channel's error-context notifier page in FBMEM before
+	 * the channel alloc so every variant carries the memdesc. A failure is
+	 * logged and tolerated: the channel then keeps the zeroed (server
+	 * ADDR_UNKNOWN/SYSMEM) notifier and stage 4's token recovery covers the
+	 * notify rejection.
+	 */
+	ret = gsp_phys_rmcall_alloc_notifier_mem(bar0);
+	if (ret)
+		pr_warn("gsp-phys-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n",
 			ret);
-		boot_stage.rmcall_token = 0;
-	} else {
-		boot_stage.rmcall_token = token;
-	}
 
-	pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x engine=COPY0 (cid is server-assigned, request value ignored)\n",
+	ret = gsp_phys_rmcall_alloc_channel(bar0);
+	if (ret)
+		return ret;
+
+	pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n",
 		boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
 		boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant,
-		boot_stage.rmcall_token);
+		boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
+		boot_stage.rmcall_notifier_base);
 	return 0;
 }
 
@@ -5042,11 +5235,21 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma,
 }
 
 /*
- * Stage 4: GPFIFO_SCHEDULE, (retry) GET_WORK_SUBMIT_TOKEN, then the first
+ * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE ->
+ * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN - then the first
  * real work item: write the CE method block to the pushbuffer, publish the
  * GPFIFO entry and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the
  * PF channel doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem
  * semaphore word. Afterwards dst must equal src byte for byte.
+ *
+ * BIND is the precondition the TU102 token generator needs (runlist
+ * membership, kernel_fifo_tu102.c:110-116; GPFIFO_SCHEDULE alone only sets
+ * RUNLIST_SET at kernel_channel.c:3244), and SET_NOTIF_INDEX (index 3 for
+ * subdevice 0, nvidia-push-init.c:303-316) gives the FBMEM notifier write a
+ * target slot. Every control failure logs its rpc_result/status and aborts
+ * before the doorbell; a 0x40 GET_WORK_SUBMIT_TOKEN whose echoed params carry
+ * a nonzero token still proceeds (the server writes the token before the
+ * notifier update, kernel_channel.c:3471-3477).
  */
 static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 {
@@ -5056,10 +5259,13 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
 	struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
 	struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
+	u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0;	/* NVA06F engineType */
+	u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX;	/* NVC36F index */
 	u8 sched_params[2] = { 1, 0 };	/* NVA06F bEnable=1, bSkipSubmit=0 */
 	resource_size_t bar0_len;
 	u32 *pb_va, *userd, *ring_va;
 	u32 words, gpput, gpget, slot, e0, e1, poll;
+	u32 token, rpc_result, status, runlist_id, ch_id, doorbell;
 	u64 pb_dma, sem_dma, src_dma, dst_dma;
 	unsigned long deadline;
 	int ret;
@@ -5086,27 +5292,78 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 		return -EIO;
 	}
 
+	/* (a) BIND the channel to the COPY0 engine (nvidia-push-init.c:343-353). */
+	rpc_result = 0;
+	status = 0;
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
+				      GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL,
+				      &bind_params, sizeof(bind_params), 16,
+				      NULL, 0, &rpc_result, &status,
+				      "BIND_CHANNEL");
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: BIND_CHANNEL (0xa06f0104 engineType=0x%x) failed (rpc_result=0x%x status=0x%x)\n",
+		       bind_params, rpc_result, status);
+		return ret;
+	}
+
+	/* (b) GPFIFO_SCHEDULE bEnable=1, bSkipSubmit=0 (existing step). */
+	rpc_result = 0;
+	status = 0;
 	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
 				      GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE,
 				      sched_params, sizeof(sched_params), 16,
-				      NULL, 0, NULL, "GPFIFO_SCHEDULE");
-	if (ret)
+				      NULL, 0, &rpc_result, &status,
+				      "GPFIFO_SCHEDULE");
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GPFIFO_SCHEDULE (0xa06f0103 bEnable=1 bSkipSubmit=0) failed (rpc_result=0x%x status=0x%x)\n",
+		       rpc_result, status);
 		return ret;
+	}
 
-	if (!boot_stage.rmcall_token) {
-		u32 token = 0;
+	/* (c) SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (nvidia-push-init.c:303-316). */
+	rpc_result = 0;
+	status = 0;
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
+				      GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX,
+				      &notif_index, sizeof(notif_index), 16,
+				      NULL, 0, &rpc_result, &status,
+				      "SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX");
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (0xc36f010a index=%u) failed (rpc_result=0x%x status=0x%x)\n",
+		       notif_index, rpc_result, status);
+		return ret;
+	}
 
-		ret = gsp_phys_rmcall_control(
-			bar0, boot_stage.rmcall_channel,
-			GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
-			&token, sizeof(token), 28, &token, sizeof(token),
-			NULL, "GET_WORK_SUBMIT_TOKEN(after schedule)");
-		if (ret) {
-			pr_err("gsp-phys-boot: rm-call: no work-submit token after GPFIFO_SCHEDULE; refusing to ring the doorbell\n");
-			return ret;
-		}
-		boot_stage.rmcall_token = token;
+	/*
+	 * (d) GET_WORK_SUBMIT_TOKEN. The helper copies the echoed params into
+	 * token before checking status, so a nonzero token can be recovered
+	 * from a failed (0x40) reply: the server writes the token into the
+	 * caller's params before the notifier update (kernel_channel.c:3471-
+	 * 3477). A zero token on failure still aborts.
+	 */
+	token = 0;
+	rpc_result = 0;
+	status = 0;
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
+				      GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
+				      &token, sizeof(token), 28, &token,
+				      sizeof(token), &rpc_result, &status,
+				      "GET_WORK_SUBMIT_TOKEN");
+	if (ret && token != 0) {
+		pr_warn("gsp-phys-boot: rm-call: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; the server wrote it before the notifier update, proceeding to submission\n",
+			rpc_result, status, token);
+		ret = 0;
 	}
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN (0xc36f0108 after BIND/SCHEDULE/notif-index=%u) failed (rpc_result=0x%x status=0x%x token=0x%x); refusing to ring the doorbell\n",
+		       notif_index, rpc_result, status, token);
+		return ret;
+	}
+	if (!token) {
+		pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN returned a zero token; refusing to ring the doorbell\n");
+		return -EIO;
+	}
+	boot_stage.rmcall_token = token;
 
 	pb_va = gsp_phys_chunk_page_va(pb, 0);
 	ring_va = gsp_phys_chunk_page_va(ring, 0);
@@ -5142,12 +5399,26 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	dma_wmb();
 	put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES,
 			   (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
+	/*
+	 * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) |
+	 * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated
+	 * token already is that value; log its runlist/ChID decomposition and
+	 * write the formula's result (identical unless the token carries stray
+	 * high bits, which is logged).
+	 */
+	runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu;
+	ch_id = boot_stage.rmcall_token & 0xfffu;
+	doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu);
+	if (doorbell != boot_stage.rmcall_token)
+		pr_warn("gsp-phys-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n",
+			boot_stage.rmcall_token, doorbell);
 	wmb();
-	writel(boot_stage.rmcall_token, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF);
+	writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF);
 
-	pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x doorbell@0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
+	pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
 		pb_dma, words, slot, gpget, gpput, e0, e1,
-		boot_stage.rmcall_token, GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma,
+		boot_stage.rmcall_token, runlist_id, ch_id, doorbell,
+		GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma,
 		GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS);
 
 	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS);
@@ -5320,7 +5591,9 @@ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target)
 /*
  * Free every ladder chunk and clear all ladder state, so a re-prime in the
  * same module load starts from zero (no stale server handles/tokens/stage).
- * Safe on a partially built ladder.
+ * Safe on a partially built ladder. The FBMEM error-notifier is a server
+ * object like the VAS/TSG/channel (no host chunk); only its state is cleared,
+ * teardown is UNLOAD + rmmod.
  */
 static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
 {
@@ -5419,7 +5692,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rm_probe_last_function,
 		   boot_stage.rm_probe_last_payload_len);
 	seq_printf(m,
-		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x\n",
+		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x notifier=%s base=0x%llx\n",
 		   boot_stage.rmcall_ran, boot_stage.rmcall_stage,
 		   boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret,
 		   boot_stage.rmcall_rpcs, boot_stage.rmcall_client,
@@ -5427,7 +5700,9 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
 		   boot_stage.rmcall_channel,
 		   boot_stage.rmcall_channel_variant,
-		   boot_stage.rmcall_token);
+		   boot_stage.rmcall_token,
+		   boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
+		   boot_stage.rmcall_notifier_base);
 	seq_printf(m,
 		   "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n",
 		   wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0),
@@ -6271,6 +6546,12 @@ static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) == 56,
 	      "rpc_alloc_memory_v13_01 first PTE must be at payload offset 56");
 static_assert(sizeof(struct gsp_phys_rmcall_mem_desc) == 24,
 	      "NV_MEMORY_DESC_PARAMS must be 24 bytes");
+static_assert(sizeof(struct gsp_phys_rmcall_memory_alloc_params) == 128,
+	      "NV_MEMORY_ALLOCATION_PARAMS must be 128 bytes (reference ABI)");
+static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, size) == 64,
+	      "NV_MEMORY_ALLOCATION_PARAMS.size must be at offset 64");
+static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset) == 80,
+	      "NV_MEMORY_ALLOCATION_PARAMS.offset [IN/OUT] must be at offset 80");
 static_assert(sizeof(struct gsp_phys_rmcall_vas_params) == 48,
 	      "NV_VASPACE_ALLOCATION_PARAMETERS must be 48 bytes (reference ABI)");
 static_assert(sizeof(struct gsp_phys_rmcall_pdb_params) == 32,
@@ -6301,6 +6582,10 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_phys_channel_gro
 	      "NV_CHANNEL_ALLOC_PARAMS.hPhysChannelGroup must be at offset 240");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, internal_flags) == 244,
 	      "NV_CHANNEL_ALLOC_PARAMS.internalFlags must be at offset 244");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, error_notifier_mem) == 248,
+	      "NV_CHANNEL_ALLOC_PARAMS.errorNotifierMem (FBMEM on GSP) must be at offset 248");
+static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ecc_error_notifier_mem) == 272,
+	      "NV_CHANNEL_ALLOC_PARAMS.eccErrorNotifierMem must be at offset 272");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296,
 	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,