vgsp: GR ladder shared-VAS page-table install and server-managed ctx commit path

- DMA_UPDATE_PDE_2 client control installs the ladder leaf PT through the
  server's own MMU walker on the shared-management VAS (the raw-PDB entries
  are not preserved by the root migration); FLUSH_PDE_CACHE + post-install
  refill + PT diagnostics.
- srvctx registry profile also disables split VAS management; MAP_MEMORY_DMA
  is not served by GSP-RM for this client (NV_ERR_INVALID_FUNCTION), so the
  PDE control replaces it.
- shared-path context commit helper: GPU_INITIALIZE_CTX + PA-only
  GPU_PROMOTE_CTX for the server-reported (or client fallback) MAIN buffer;
  GR_GET_CTX_BUFFER_INFO sends bufferCount=64.
- Result: shared VAS + srvctx no longer wedges; channel/BIND/schedule/token
  green, PBDMA fetches the GPFIFO (GPGet=1); remaining wall is
  ROBUST_CHANNEL_GR_FAULT_DURING_CTXSW (0x2c).

1 files changed, 291 insertions, 8 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index bead218..aed4c67 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -243,6 +243,7 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL	0xa06f0104u
 #define GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE	0xa06c0101u
 #define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY	0x801813u
+#define GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2		0x80180fu
 #define GSP_PHYS_RMCALL_ENGINE_COPY0		0x9u
 
 /*
@@ -1319,6 +1320,15 @@ static struct gsp_phys_boot_stage {
 	u32 rmcall_ctx_pages;		/* 4 KB pages mapped */
 	u32 rmcall_ctx_pt_pages;	/* small-PT pages used */
 	/*
+	 * Shared-management path: the server-managed MAIN context buffer
+	 * reported by GR_GET_CTX_BUFFER_INFO; the client initialize+promote
+	 * step installs it (PA-only) so the GR ucode can load a valid context.
+	 */
+	bool rmcall_srv_main_valid;
+	u64 rmcall_srv_main_pa;
+	u64 rmcall_srv_main_size;
+	u32 rmcall_srv_main_aperture;
+	/*
 	 * Mode 13: the client PATCH context buffer (PA-only promote). Prepared
 	 * lazily in the promote step from the engine-context geometry query
 	 * (engineId 0x10); freed by gsp_phys_rmcall_fini().
@@ -1960,6 +1970,10 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_ctx_size = 0;
 	boot_stage.rmcall_ctx_pages = 0;
 	boot_stage.rmcall_ctx_pt_pages = 0;
+	boot_stage.rmcall_srv_main_valid = false;
+	boot_stage.rmcall_srv_main_pa = 0;
+	boot_stage.rmcall_srv_main_size = 0;
+	boot_stage.rmcall_srv_main_aperture = 0;
 	boot_stage.rmcall_patch_prepared = false;
 	boot_stage.rmcall_patch_pa = 0;
 	boot_stage.rmcall_patch_size = 0;
@@ -5732,6 +5746,39 @@ struct gsp_phys_rmcall_alloc_memory_req {
 	__le64 pte[];
 };
 
+/*
+ * NV0080_CTRL_DMA_UPDATE_PDE_2_PARAMS (ctrl0080dma.h:676-684): the
+ * kernel-client control that installs one PDE pointing at a client-supplied
+ * leaf page table. It is the supported way for a raw-RPC client to add a
+ * mapping to a shared-management VAS, where the server owns the page tables:
+ * with pPdeBuffer=NULL the server's mmuWalkMap allocates every higher level
+ * and writes the entry into its own page directory (gpu_vaspace.c
+ * gvaspaceUpdatePde2_IMPL). MAP_MEMORY_DMA (fn14) is a vGPU-guest RPC and is
+ * answered NV_ERR_INVALID_FUNCTION by GSP-RM for this client.
+ */
+struct gsp_phys_rmcall_update_pde2_pt_params {
+	__le64 phys_addr;
+	__le32 num_entries;	/* deprecated */
+	__le32 aperture;
+};
+
+struct gsp_phys_rmcall_update_pde2_params {
+	__le32 pde_index;
+	__le32 flags;
+	struct gsp_phys_rmcall_update_pde2_pt_params pt[2];
+	__le32 h_va_space;
+	__le32 pad1;
+	__le64 pde_buffer;	/* 0 = write the server's internal PD */
+	__le32 sub_device_id;
+	__le32 pad2;
+};
+
+#define GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH	1u
+#define GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID	0u
+#define GSP_PHYS_RMCALL_PDE2_APERTURE_VIDMEM	1u
+#define GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH	2u
+#define GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_NONCOH 3u
+
 /* NV_MEMORY_DESC_PARAMS (alloc/alloc_channel.h:37-42). */
 struct gsp_phys_rmcall_mem_desc {
 	__le64 base;
@@ -6280,6 +6327,97 @@ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index)
 	return 0;
 }
 
+static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
+				   const void *params, u32 params_len,
+				   u32 reply_min, void *out, u32 out_len,
+				   u32 *out_rpc_result, u32 *out_status,
+				   const char *what);
+static void gsp_phys_rmcall_pdb_build(void);
+
+/*
+ * Shared-management VAS: install the ladder's page-table window through the
+ * server's own MMU walker. NV0080_CTRL_CMD_DMA_UPDATE_PDE_2 (0x80180f,
+ * gpu_vaspace.c gvaspaceUpdatePde2_IMPL) takes the system-physical base of a
+ * leaf page table, installs its PDE at pdeIndex in the level directly above
+ * the 4 KiB PTs, and with pPdeBuffer=NULL calls mmuWalkMap so the server
+ * allocates every higher page-table level and writes the entry into its own
+ * page directory. That is the supported client map path for a shared VAS:
+ * the raw-PTE gsp_phys_rmcall_pdb_build entries are not preserved by the
+ * server's root migration, which is why the engine faulted FAULT_PDE at the
+ * GPFIFO VA. One PDE covers VA 0x0..0x1fffff, where all ladder VAs live
+ * (GPFIFO/PB/SRC/DST/SEM), and their PTEs are already in the PT chunk page
+ * written by the PDB build. The GR context buffer needs no client mapping:
+ * with gsp_phys_srv_ctxbuf the server allocates and commits it itself.
+ */
+static int gsp_phys_rmcall_map_pt_window(void __iomem *bar0)
+{
+	struct gsp_phys_boot_chunk *pt =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
+	struct gsp_phys_rmcall_update_pde2_params p;
+	u32 rpc_result = 0, status = 0;
+	u64 pt_pa = (u64)gsp_phys_chunk_page_dma(pt, 0);
+	int ret;
+
+	if (!boot_stage.rmcall_vas) {
+		pr_err("vgsp-boot: rm-call: DMA_UPDATE_PDE_2: no ladder VAS handle; refusing\n");
+		return -EINVAL;
+	}
+	if (!boot_stage.rmcall_mem_registered[GSP_PHYS_RMCALL_BUF_PT]) {
+		pr_err("vgsp-boot: rm-call: DMA_UPDATE_PDE_2: PT buffer is not registered; refusing\n");
+		return -EINVAL;
+	}
+	memset(&p, 0, sizeof(p));
+	p.pde_index = 0;	/* the 2 MiB window holding every ladder VA */
+	p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
+	p.pt[0].phys_addr = cpu_to_le64(pt_pa);
+	p.pt[0].num_entries = 0;	/* deprecated */
+	p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
+	p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
+	p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
+	p.pde_buffer = 0;	/* write the server's internal PD */
+	p.sub_device_id = 0;
+	pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 pre: pt[0x100]=0x%016llx pt[0x101]=0x%016llx\n",
+		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
+				   0x100u * 8),
+		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
+				   0x101u * 8));
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
+			GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
+			&p, sizeof(p), sizeof(p), &p, sizeof(p),
+			&rpc_result, &status, "DMA_UPDATE_PDE_2");
+	if (ret) {
+		pr_err("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 rejected: status=0x%x rpc_result=0x%x ret=%d; the engine cannot fetch the GPFIFO on this shared VAS and the ladder aborts with no retry\n",
+		       status, rpc_result, ret);
+		return ret;
+	}
+	pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 accepted: pdeIndex=0 ptPa=0x%llx aperture=SYS_COH hVas=0x%x status=0x%x echoed{flags=0x%x pt0={pa=0x%llx num=%u apt=%u} pt1={pa=0x%llx num=%u apt=%u}} (one PDE covers VA 0x0..0x1fffff: GPFIFO/PB/SRC/DST/SEM)\n",
+		pt_pa, boot_stage.rmcall_vas, status,
+		le32_to_cpu(p.flags),
+		le64_to_cpu(p.pt[0].phys_addr),
+		le32_to_cpu(p.pt[0].num_entries),
+		le32_to_cpu(p.pt[0].aperture),
+		le64_to_cpu(p.pt[1].phys_addr),
+		le32_to_cpu(p.pt[1].num_entries),
+		le32_to_cpu(p.pt[1].aperture));
+	pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 post: pt[0x100]=0x%016llx pt[0x101]=0x%016llx\n",
+		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
+				   0x100u * 8),
+		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
+				   0x101u * 8));
+	/*
+	 * Belt-and-braces: rewrite the PT contents after the walker install
+	 * (the install must not be allowed to leave the externally supplied PT
+	 * page zeroed). All PAs are unchanged, so this only restores values.
+	 */
+	gsp_phys_rmcall_pdb_build();
+	pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 refill: pt[0x100]=0x%016llx pt[0x101]=0x%016llx\n",
+		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
+				   0x100u * 8),
+		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
+				   0x101u * 8));
+	return 0;
+}
+
 /*
  * Allocate the channel error-context notifier in FBMEM. On GSP the channel
  * notify path rejects a non-FBMEM error-context memdesc with
@@ -8628,6 +8766,10 @@ static void gsp_phys_rmcall_gr_ctx_info(void __iomem *bar0)
 	}
 	p->h_user_client = cpu_to_le32(boot_stage.rmcall_client);
 	p->h_channel = cpu_to_le32(boot_stage.rmcall_channel);
+	/* bufferCount is IN/OUT: the server rejects a max smaller than the
+	 * context's buffer count (kgrctxGetCtxBufferInfo -> INVALID_ARGUMENT,
+	 * the 0x1f every earlier run saw). */
+	p->buffer_count = cpu_to_le32(64);
 	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
 			GSP_PHYS_RMCALL_CTRL_GR_GET_CTX_BUFFER_INFO,
 			p, sizeof(*p), 24 + sizeof(*p),
@@ -8644,21 +8786,135 @@ static void gsp_phys_rmcall_gr_ctx_info(void __iomem *bar0)
 	shown = min_t(u32, count, 4u);
 	pr_info("vgsp-boot: rm-call: GR_GET_CTX_BUFFER_INFO: bufferCount=%u (showing %u)\n",
 		count, shown);
-	for (i = 0; i < shown; i++) {
+	for (i = 0; i < count; i++) {
 		const struct gsp_phys_rmcall_ctx_buffer_info *ci =
 			&p->ctx_buffer_info[i];
 
-		pr_info("vgsp-boot: rm-call: GR ctx[%u]: bufferType=%u physAddr=0x%llx aperture=%u size=0x%llx pageCount=0x%llx alignment=0x%llx contiguous=%u\n",
-			i, le32_to_cpu(ci->buffer_type),
-			le64_to_cpu(ci->phys_addr),
-			le32_to_cpu(ci->aperture), le64_to_cpu(ci->size),
-			le64_to_cpu(ci->page_count),
-			le64_to_cpu(ci->alignment),
-			le32_to_cpu(ci->b_is_contigous));
+		if (i < shown)
+			pr_info("vgsp-boot: rm-call: GR ctx[%u]: bufferType=%u physAddr=0x%llx aperture=%u size=0x%llx pageCount=0x%llx alignment=0x%llx contiguous=%u\n",
+				i, le32_to_cpu(ci->buffer_type),
+				le64_to_cpu(ci->phys_addr),
+				le32_to_cpu(ci->aperture),
+				le64_to_cpu(ci->size),
+				le64_to_cpu(ci->page_count),
+				le64_to_cpu(ci->alignment),
+				le32_to_cpu(ci->b_is_contigous));
+		/* MAIN is bufferType 0 (NV2080_CTRL_GR_CTX_BUFFER_INFO enum). */
+		if (le32_to_cpu(ci->buffer_type) == 0 &&
+		    !boot_stage.rmcall_srv_main_valid) {
+			boot_stage.rmcall_srv_main_valid = true;
+			boot_stage.rmcall_srv_main_pa =
+				le64_to_cpu(ci->phys_addr);
+			boot_stage.rmcall_srv_main_size =
+				le64_to_cpu(ci->size);
+			boot_stage.rmcall_srv_main_aperture =
+				le32_to_cpu(ci->aperture);
+			pr_info("vgsp-boot: rm-call: server MAIN ctx buffer: pa=0x%llx size=0x%llx aperture=%u\n",
+				boot_stage.rmcall_srv_main_pa,
+				boot_stage.rmcall_srv_main_size,
+				boot_stage.rmcall_srv_main_aperture);
+		}
 	}
 	kvfree(p);
 }
 
+/*
+ * Shared-management client commit: with gsp_phys_srv_ctxbuf the server
+ * allocated/booked/mapped the ctx buffers during the TURING_COMPUTE_A
+ * construct, but the engine still faults during the GR context switch
+ * (ROBUST_CHANNEL_GR_FAULT_DURING_CTXSW, 0x2c) because the context was never
+ * initialized. Send the two client-side controls that do it for the
+ * server-reported MAIN buffer: GPU_INITIALIZE_CTX installs the PA/size/attr
+ * memdesc, then GPU_PROMOTE_CTX with a single PA-only MAIN entry (the mode-2/
+ * mode-12 shape the server accepts) commits it. Runs only when the shared
+ * path reported a MAIN buffer.
+ */
+static int gsp_phys_rmcall_srv_ctx_commit(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_init_ctx_params *ic;
+	struct gsp_phys_rmcall_promote_params *p;
+	u32 rpc_result = 0, status = 0;
+	u32 pa_attr;
+	u64 main_pa, main_size;
+	int ret;
+
+	if (boot_stage.rmcall_srv_main_valid) {
+		main_pa = boot_stage.rmcall_srv_main_pa;
+		main_size = boot_stage.rmcall_srv_main_size;
+		pa_attr = (boot_stage.rmcall_srv_main_aperture == 0) ?
+			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB :
+			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
+		pr_info("vgsp-boot: rm-call: committing the SERVER-reported MAIN ctx buffer pa=0x%llx size=0x%llx\n",
+			main_pa, main_size);
+	} else {
+		/*
+		 * GR_GET_CTX_BUFFER_INFO still refused (0x1f even with a 64
+		 * bufferCount); fall back to the client MAIN buffer prepared by
+		 * gsp_phys_rmcall_gr_ctx_prepare with the mode-12 PA-only
+		 * shape the server accepted on the external V5 path. Its VA
+		 * window is not installed on this shared VAS, so the entry is
+		 * PA-only bNonmapped=1 (the save/restore image is addressed by
+		 * PA).
+		 */
+		main_pa = boot_stage.rmcall_ctx_pa;
+		main_size = boot_stage.rmcall_ctx_size;
+		pa_attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
+		pr_warn("vgsp-boot: rm-call: server MAIN ctx buffer was not reported; committing the CLIENT MAIN buffer pa=0x%llx size=0x%llx PA-only instead\n",
+			main_pa, main_size);
+	}
+
+	ic = kvzalloc(sizeof(*ic), GFP_KERNEL);
+	if (!ic)
+		return -ENOMEM;
+	ic->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+	ic->h_client = cpu_to_le32(0);
+	ic->ch_id = cpu_to_le32(0);
+	ic->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
+	ic->h_object = cpu_to_le32(boot_stage.rmcall_channel);
+	ic->h_virt_memory = cpu_to_le32(0);
+	ic->phys_address = cpu_to_le64(main_pa);
+	ic->phys_attr = cpu_to_le32(pa_attr);
+	ic->h_dma_handle = cpu_to_le32(0);
+	ic->index = cpu_to_le32(0);
+	ic->size = cpu_to_le64(main_size);
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
+			GSP_PHYS_RMCALL_CTRL_GPU_INITIALIZE_CTX,
+			ic, sizeof(*ic), 24, NULL, 0,
+			&rpc_result, &status,
+			"GPU_INITIALIZE_CTX (server MAIN)");
+	kvfree(ic);
+	pr_info("vgsp-boot: rm-call: MAIN INITIALIZE_CTX: pa=0x%llx size=0x%llx physAttr=0x%x status=0x%x rpc_result=0x%x\n",
+		main_pa, main_size, pa_attr, status, rpc_result);
+	if (ret)
+		return ret;
+
+	p = kvzalloc(sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+	p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+	p->h_client = cpu_to_le32(0);
+	p->ch_id = cpu_to_le32(0);
+	p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
+	p->h_object = cpu_to_le32(boot_stage.rmcall_channel);
+	p->h_virt_memory = cpu_to_le32(0);
+	p->virt_address = cpu_to_le64(0);
+	p->size = cpu_to_le64(0);
+	p->entry_count = cpu_to_le32(1);
+	gsp_phys_rmcall_promote_entry_fill(p, 0, 0,
+			main_pa, 0, main_size, pa_attr, 1, 1);
+	rpc_result = 0;
+	status = 0;
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
+			GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+			p, sizeof(*p), 24, NULL, 0,
+			&rpc_result, &status,
+			"GPU_PROMOTE_CTX (server MAIN PA-only)");
+	pr_info("vgsp-boot: rm-call: MAIN PROMOTE_CTX: entry{bufferId=MAIN pa=0x%llx size=0x%llx physAttr=0x%x bInitialize=1 bNonmapped=1} status=0x%x rpc_result=0x%x ret=%d\n",
+		main_pa, main_size, pa_attr, status, rpc_result, ret);
+	kvfree(p);
+	return ret;
+}
+
 
 /*
  * Stage 3: explicit VAS with its page directory, then channel group (COPY0,
@@ -8930,6 +9186,33 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 		gsp_phys_rmcall_gr_ctx_info(bar0);
 	}
 
+	/*
+	 * Shared-management VAS: the server owns the lower page tables, so the
+	 * hand-built PTEs of gsp_phys_rmcall_pdb_build are not preserved by its
+	 * root migration and the engine faults FAULT_PDE. Install the ladder's
+	 * 2 MiB page-table window through the server's own MMU walker
+	 * (NV0080_CTRL_CMD_DMA_UPDATE_PDE_2) after its internal context
+	 * mappings (which run during the TURING_COMPUTE_A construct on a GR
+	 * ladder). The external-VAS path keeps the raw PDB (client-owned page
+	 * tables, no server booking).
+	 */
+	if (gsp_phys_rmcall_vas_shared_value()) {
+		if (gsp_phys_rmcall_engine_is_gr()) {
+			ret = gsp_phys_rmcall_srv_ctx_commit(bar0);
+			if (ret) {
+				pr_err("vgsp-boot: rm-call: stage 3 aborted: shared-VAS server MAIN initialize+promote failed (%d); the GR context stays uninitialized and the ladder is not retried\n",
+				       ret);
+				return ret;
+			}
+		}
+		ret = gsp_phys_rmcall_map_pt_window(bar0);
+		if (ret) {
+			pr_err("vgsp-boot: rm-call: stage 3 aborted: shared-VAS page-table window mapping failed (%d); without the GPFIFO/pushbuffer mappings the engine faults FAULT_PDE and the ladder is not retried\n",
+			       ret);
+			return ret;
+		}
+	}
+
 	if (gsp_phys_rmcall_engine_is_gr())
 		scnprintf(promote_suffix, sizeof(promote_suffix),
 			  " gr_promote_param=%u step=%u entrySize=0x%x",