gsp-phys: submit the CE copy in virtual mode through the mapped VA window

SET_SEMAPHORE_A/B always take a GPU VA and the channel is USER, so the
release cannot target a raw DMA address and physical-mode src/dst is not
permitted. Map the src/dst/semem pages at VA 0x102000/0x103000/0x104000
(small-PT slots 258/259/260) in the existing channel PDB, push the semaphore
VA, drop the SET_SRC/DST_PHYS_MODE pairs and clear LAUNCH_DMA bits 12/13
(0x18d: Pipelined|Flush|Release-one-word|Pitch). The block is now 13 method
pairs / 26 dwords; GPFIFO/PUT/doorbell/poll/memcmp are unchanged.

1 files changed, 109 insertions, 58 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 4e7fd6f..943e5f2 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -287,17 +287,23 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID		1u
 
 /*
- * Channel page-directory GPU VA window. The GPFIFO and the pushbuffer are
- * fetched by GPU VA (the GPFIFO from gpFifoOffset, the pushbuffer from the
- * GPFIFO ring entry GET), so the channel's external root must translate them.
- * Both pages live in one 2 MB PD0 slot below the VAS vaSize limit: with
+ * Channel page-directory GPU VA window. The GPFIFO, the pushbuffer and the
+ * CE copy's src/dst/semaphore pages are all fetched by GPU VA (the GPFIFO
+ * from gpFifoOffset, the pushbuffer from the GPFIFO ring entry GET, the CE
+ * operands from the method block): SET_SEMAPHORE_A/B are always GPU VAs with
+ * no mode bit (clc5b5.h:39-44; uvm_pascal_ce.c:42-63) and the user channel's
+ * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All five
+ * pages live in one 2 MB PD0 slot below the VAS vaSize limit: with
  * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0
- * and the two 4 KB pages take small-PT slots 0x100/0x101
+ * and the five 4 KB pages take small-PT slots 0x100..0x104
  * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204).
  */
 #define GSP_PHYS_RMCALL_VA_BASE		0x00100000ULL
 #define GSP_PHYS_RMCALL_VA_GPFIFO	GSP_PHYS_RMCALL_VA_BASE
 #define GSP_PHYS_RMCALL_VA_PB		(GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE)
+#define GSP_PHYS_RMCALL_VA_SRC		(GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE)
+#define GSP_PHYS_RMCALL_VA_DST		(GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE)
+#define GSP_PHYS_RMCALL_VA_SEM		(GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE)
 
 /*
  * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
@@ -316,13 +322,11 @@ bool gsp_phys_boot_requested(void)
  * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 and ADDR_FBMEM=2
  * (kernel/vgpu/rm_plugin_shared_code.h:66-67) and NV_MEMORY_UNCACHED=1
  * (kernel/os/nv_memory_type.h:31), which is the default RM system-memory CPU
- * cache attribute (system_mem.c:73-74) and selects the NONCOHERENT_SYSMEM CE
- * aperture (clc5b5.h:56-60).
+ * cache attribute (system_mem.c:73-74).
  */
 #define GSP_PHYS_RMCALL_ADDR_SYSMEM		1u
 #define GSP_PHYS_RMCALL_ADDR_FBMEM		2u
 #define GSP_PHYS_RMCALL_CACHE_UNCACHED		1u
-#define GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM	2u
 
 /* NVOS02 flags: PHYSICALITY field 7:4, NONCONTIGUOUS=1 -> 1 << 4 (nvos.h:189-191). */
 #define GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS	(1u << 4)
@@ -420,20 +424,20 @@ enum gsp_phys_rmcall_buf {
 /*
  * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from
  * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248,
- * SET_SRC_PHYS_MODE 0x260, SET_DST_PHYS_MODE 0x264, LAUNCH_DMA 0x300,
- * OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/LOWER 0x408/0x40c,
- * PITCH_IN/OUT 0x410/0x414, LINE_LENGTH_IN/LINE_COUNT 0x418/0x41c.
- * The method order and the physical-mode/pitch/launch encoding mirror
- * _ceChannelPushMethodsBlock_GM107 (gpu/mem_mgr/arch/maxwell/
- * mem_utils_gm107.c:1968-2118) and the semaphore payload block
- * (gpu/mem_mgr/channel_utils.c:812-820).
+ * LAUNCH_DMA 0x300, OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/
+ * LOWER 0x408/0x40c, PITCH_IN/OUT 0x410/0x414, LINE_LENGTH_IN/LINE_COUNT
+ * 0x418/0x41c. The block is the all-virtual user-channel shape: the
+ * semaphore methods carry a GPU VA only (no mode bit, clc5b5.h:39-44;
+ * uvm_pascal_ce.c:42-63), the src/dst VAs are mapped in the channel VAS and
+ * no SET_SRC/DST_PHYS_MODE pair is pushed (uvm_maxwell_ce.c:179-205); the
+ * method order mirrors the RM's own block
+ * (gpu/mem_mgr/arch/maxwell/mem_utils_gm107.c:1968-2118) and the semaphore
+ * payload block (gpu/mem_mgr/channel_utils.c:812-820).
  */
 #define GSP_PHYS_RMCALL_CE_SET_OBJECT		0x000u
 #define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A	0x240u
 #define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B	0x244u
 #define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD	0x248u
-#define GSP_PHYS_RMCALL_CE_SET_SRC_PHYS_MODE	0x260u
-#define GSP_PHYS_RMCALL_CE_SET_DST_PHYS_MODE	0x264u
 #define GSP_PHYS_RMCALL_CE_LAUNCH_DMA		0x300u
 #define GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER	0x400u
 #define GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER	0x404u
@@ -445,14 +449,13 @@ enum gsp_phys_rmcall_buf {
 #define GSP_PHYS_RMCALL_CE_LINE_COUNT		0x41cu
 
 /* LAUNCH_DMA: PIPELINED 1:0=1, FLUSH_ENABLE bit2, SEMAPHORE_TYPE 4:3=
- * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE PHYSICAL
- * bits12/13 (clc5b5.h:69-108); the physical memcopy sets exactly this shape
- * (mem_utils_gm107.c:2082-2097), including FLUSH_ENABLE when the payload
- * semaphore is requested. */
+ * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE VIRTUAL
+ * bits12/13=0 (clc5b5.h:69-108); exactly the UVM user-channel pitch copy
+ * with a one-word release (uvm_maxwell_ce.c:277-291, uvm_pascal_ce.c:42-63):
+ * 0x18d. */
 #define GSP_PHYS_RMCALL_LAUNCH_DMA			\
 	(0x1u | (0x1u << 2) | (0x1u << 3) |		\
-	 (0x1u << 7) | (0x1u << 8) |			\
-	 (0x1u << 12) | (0x1u << 13))
+	 (0x1u << 7) | (0x1u << 8))
 
 /*
  * PCIe config-space mirror window, as carried in GspSystemInfo. The field is
@@ -930,6 +933,9 @@ static struct gsp_phys_boot_stage {
 	u64 rmcall_va_base;		/* channel PDB VA window base */
 	u64 rmcall_va_gpfifo;		/* GPFIFO GPU VA in that window */
 	u64 rmcall_va_pb;		/* pushbuffer GPU VA in that window */
+	u64 rmcall_va_src;		/* CE source GPU VA in that window */
+	u64 rmcall_va_dst;		/* CE destination GPU VA in that window */
+	u64 rmcall_va_sem;		/* CE semaphore GPU VA in that window */
 	u64 rmcall_pdb_root;		/* root entry -> PD2 */
 	u64 rmcall_pdb_pd2;		/* PD2 entry -> PD1 */
 	u64 rmcall_pdb_pd1;		/* PD1 entry -> PD0 */
@@ -1454,6 +1460,9 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_va_base = 0;
 	boot_stage.rmcall_va_gpfifo = 0;
 	boot_stage.rmcall_va_pb = 0;
+	boot_stage.rmcall_va_src = 0;
+	boot_stage.rmcall_va_dst = 0;
+	boot_stage.rmcall_va_sem = 0;
 	boot_stage.rmcall_pdb_root = 0;
 	boot_stage.rmcall_pdb_pd2 = 0;
 	boot_stage.rmcall_pdb_pd1 = 0;
@@ -5151,15 +5160,16 @@ static u64 gsp_phys_rmcall_pte(u64 pa)
 
 /*
  * Build the channel's page-directory hierarchy around the 2 MB VA window
- * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO and pushbuffer sysmem pages:
+ * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO, pushbuffer and the CE copy's
+ * src/dst/semaphore sysmem pages:
  *
  *   root/PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 dual [28:21]
  *     -> small PT [20:12]
  *
  * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,143-161). The window
- * puts every parent index at 0 and the two pages at small-PT slots
- * 0x100/0x101, so only root[0], PD2[0], PD1[0], the PD0[0] small half and
- * those two PT slots are written; the root chunk itself is the
+ * puts every parent index at 0 and the five pages at small-PT slots
+ * 0x100..0x104, so only root[0], PD2[0], PD1[0], the PD0[0] small half and
+ * those five PT slots are written; the root chunk itself is the
  * DMA_SET_PAGE_DIRECTORY payload. The external root merely stores the
  * memdesc, so filling before the channel alloc (after a successful
  * SET_PAGE_DIRECTORY) is equivalent to filling before the control.
@@ -5176,9 +5186,19 @@ static void gsp_phys_rmcall_pdb_build(void)
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD0];
 	struct gsp_phys_boot_chunk *pt =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
+	struct gsp_phys_boot_chunk *src =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
+	struct gsp_phys_boot_chunk *dst =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
+	struct gsp_phys_boot_chunk *sem =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
 	u32 gpfifo_idx = (u32)((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ffu);
 	u32 pb_idx = (u32)((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ffu);
-	u64 root_ent, pd2_ent, pd1_ent, pd0_small, pt_gpfifo, pt_pb;
+	u32 src_idx = (u32)((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ffu);
+	u32 dst_idx = (u32)((GSP_PHYS_RMCALL_VA_DST >> 12) & 0x1ffu);
+	u32 sem_idx = (u32)((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ffu);
+	u64 root_ent, pd2_ent, pd1_ent, pd0_small;
+	u64 pt_gpfifo, pt_pb, pt_src, pt_dst, pt_sem;
 
 	root_ent = gsp_phys_rmcall_pde(
 		(u64)gsp_phys_chunk_page_dma(pd2, 0));
@@ -5195,6 +5215,12 @@ static void gsp_phys_rmcall_pdb_build(void)
 	pt_pb = gsp_phys_rmcall_pte(
 		(u64)gsp_phys_chunk_page_dma(
 			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0));
+	pt_src = gsp_phys_rmcall_pte(
+		(u64)gsp_phys_chunk_page_dma(src, 0));
+	pt_dst = gsp_phys_rmcall_pte(
+		(u64)gsp_phys_chunk_page_dma(dst, 0));
+	pt_sem = gsp_phys_rmcall_pte(
+		(u64)gsp_phys_chunk_page_dma(sem, 0));
 
 	put_unaligned_le64(root_ent, gsp_phys_chunk_page_va(root, 0));
 	put_unaligned_le64(pd2_ent, gsp_phys_chunk_page_va(pd2, 0));
@@ -5209,11 +5235,23 @@ static void gsp_phys_rmcall_pdb_build(void)
 	put_unaligned_le64(pt_pb,
 			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
 			   (size_t)pb_idx * 8);
+	put_unaligned_le64(pt_src,
+			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+			   (size_t)src_idx * 8);
+	put_unaligned_le64(pt_dst,
+			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+			   (size_t)dst_idx * 8);
+	put_unaligned_le64(pt_sem,
+			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+			   (size_t)sem_idx * 8);
 	dma_wmb();
 
 	boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE;
 	boot_stage.rmcall_va_gpfifo = GSP_PHYS_RMCALL_VA_GPFIFO;
 	boot_stage.rmcall_va_pb = GSP_PHYS_RMCALL_VA_PB;
+	boot_stage.rmcall_va_src = GSP_PHYS_RMCALL_VA_SRC;
+	boot_stage.rmcall_va_dst = GSP_PHYS_RMCALL_VA_DST;
+	boot_stage.rmcall_va_sem = GSP_PHYS_RMCALL_VA_SEM;
 	boot_stage.rmcall_pdb_root = root_ent;
 	boot_stage.rmcall_pdb_pd2 = pd2_ent;
 	boot_stage.rmcall_pdb_pd1 = pd1_ent;
@@ -5221,9 +5259,11 @@ static void gsp_phys_rmcall_pdb_build(void)
 	boot_stage.rmcall_pt_gpfifo = pt_gpfifo;
 	boot_stage.rmcall_pt_pb = pt_pb;
 
-	pr_info("gsp-phys-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n",
+	pr_info("gsp-phys-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n",
 		GSP_PHYS_RMCALL_VA_BASE, GSP_PHYS_RMCALL_VA_GPFIFO,
-		GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO,
+		GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_SRC,
+		GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM,
+		GSP_PHYS_RMCALL_VA_GPFIFO,
 		(u64)gsp_phys_chunk_page_dma(root, 0),
 		(u64)gsp_phys_chunk_page_dma(pd2, 0),
 		(u64)gsp_phys_chunk_page_dma(pd1, 0),
@@ -5239,6 +5279,10 @@ static void gsp_phys_rmcall_pdb_build(void)
 		pb_idx, pt_pb,
 		(u64)gsp_phys_chunk_page_dma(
 			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0));
+	pr_info("gsp-phys-boot: rm-call: channel PDB CE PT: pt[%u]=0x%016llx (src dma=0x%llx) pt[%u]=0x%016llx (dst dma=0x%llx) pt[%u]=0x%016llx (sem dma=0x%llx)\n",
+		src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0),
+		dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0),
+		sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0));
 }
 
 /*
@@ -5320,9 +5364,10 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 		return ret;
 	}
 	/*
-	 * The server only stored our root memdesc; the GPFIFO and pushbuffer
-	 * fetches translate through it, so fill the hierarchy before the
-	 * channel alloc commits the instance block's page-directory base.
+	 * The server only stored our root memdesc; the GPFIFO, pushbuffer and
+	 * CE src/dst/semaphore fetches translate through it, so fill the
+	 * hierarchy before the channel alloc commits the instance block's
+	 * page-directory base.
 	 */
 	gsp_phys_rmcall_pdb_build();
 
@@ -5383,44 +5428,43 @@ static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data)
 
 /*
  * CE method block for one 4 KB sysmem-to-sysmem pitch copy with a release
- * semaphore, mirroring _ceChannelPushMethodsBlock_GM107
- * (gpu/mem_mgr/arch/maxwell/mem_utils_gm107.c:1968-2118) with the semaphore
- * payload block from channel_utils.c:812-820 and the physical aperture choice
- * (system_mem.c:73-74 -> NONCOHERENT_SYSMEM). Returns the block size in
- * dwords (<= GPFIFO entry LENGTH field, bits 30:10).
+ * semaphore, in the all-virtual user-channel shape: the semaphore methods
+ * always carry a GPU VA (no mode bit; uvm_pascal_ce.c:42-63) and the src/dst
+ * VAs translate through the channel PDB, so no SET_SRC/DST_PHYS_MODE pair is
+ * pushed (uvm_maxwell_ce.c:179-205). The method order mirrors
+ * _ceChannelPushMethodsBlock_GM107 (gpu/mem_mgr/arch/maxwell/
+ * mem_utils_gm107.c:1968-2118) and the semaphore payload block
+ * (channel_utils.c:812-820). Returns the block size in dwords (13 method
+ * pairs = 26 dwords, <= GPFIFO entry LENGTH field, bits 30:10).
  */
-static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma,
-					 u64 dst_dma)
+static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va,
+					 u64 dst_va)
 {
 	u32 *p = pb;
 
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_OBJECT,
 				 GSP_PHYS_RMCALL_CLASS_CE);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A,
-				 (u32)(sem_dma >> 32));
+				 (u32)(sem_va >> 32));
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B,
-				 (u32)sem_dma);
+				 (u32)sem_va);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD,
 				 GSP_PHYS_RMCALL_SEM_PAYLOAD);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER,
-				 (u32)(dst_dma >> 32));
+				 (u32)(dst_va >> 32));
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER,
-				 (u32)dst_dma);
+				 (u32)dst_va);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_OUT,
 				 GSP_PHYS_RMCALL_COPY_SIZE);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER,
-				 (u32)(src_dma >> 32));
+				 (u32)(src_va >> 32));
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER,
-				 (u32)src_dma);
+				 (u32)src_va);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_IN,
 				 GSP_PHYS_RMCALL_COPY_SIZE);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN,
 				 GSP_PHYS_RMCALL_COPY_SIZE);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_COUNT, 1);
-	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SRC_PHYS_MODE,
-				 GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM);
-	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_DST_PHYS_MODE,
-				 GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LAUNCH_DMA,
 				 GSP_PHYS_RMCALL_LAUNCH_DMA);
 	return (u32)(p - pb);
@@ -5603,7 +5647,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	src_dma = (u64)gsp_phys_chunk_page_dma(src, 0);
 	dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0);
 
-	words = gsp_phys_rmcall_build_methods(pb_va, sem_dma, src_dma, dst_dma);
+	words = gsp_phys_rmcall_build_methods(pb_va, GSP_PHYS_RMCALL_VA_SEM,
+					      GSP_PHYS_RMCALL_VA_SRC,
+					      GSP_PHYS_RMCALL_VA_DST);
 
 	gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
 	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
@@ -5651,8 +5697,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	wmb();
 	writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF);
 
-	pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
+	pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
 		pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO,
+		GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST,
+		GSP_PHYS_RMCALL_VA_SEM,
 		words, slot, gpget, gpput, e0, e1,
 		boot_stage.rmcall_token, runlist_id, ch_id, doorbell,
 		GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma,
@@ -5941,9 +5989,10 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
 		   boot_stage.rmcall_notifier_base);
 	seq_printf(m,
-		   "rm_call_userd: userd=sysmem (host-writable) | rm_call_pdb: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx\n",
+		   "rm_call_userd: userd=sysmem (host-writable) | rm_call_pdb: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx\n",
 		   boot_stage.rmcall_va_base, boot_stage.rmcall_va_gpfifo,
-		   boot_stage.rmcall_va_pb);
+		   boot_stage.rmcall_va_pb, boot_stage.rmcall_va_src,
+		   boot_stage.rmcall_va_dst, boot_stage.rmcall_va_sem);
 	seq_printf(m,
 		   "rm_call_pdb_entries: root=0x%016llx pd2=0x%016llx pd1=0x%016llx pd0Small=0x%016llx ptGpfifo=0x%016llx ptPb=0x%016llx\n",
 		   boot_stage.rmcall_pdb_root, boot_stage.rmcall_pdb_pd2,
@@ -6850,18 +6899,20 @@ static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
 	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
 	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
 	      "rm-call memory handles must stay inside the RM client handle range");
-static_assert(GSP_PHYS_RMCALL_VA_PB + GSP_PHYS_PAGE_SIZE <=
+static_assert(GSP_PHYS_RMCALL_VA_SEM + GSP_PHYS_PAGE_SIZE <=
 	      GSP_PHYS_RMCALL_VAS_SIZE,
 	      "the channel PDB VA window must stay below the VAS vaSize limit");
 static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 &&
 	      ((GSP_PHYS_RMCALL_VA_BASE >> 38) & 0x1ff) == 0 &&
 	      ((GSP_PHYS_RMCALL_VA_BASE >> 29) & 0x1ff) == 0 &&
 	      ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 &&
-	      ((GSP_PHYS_RMCALL_VA_PB >> 21) & 0xff) == 0,
+	      ((GSP_PHYS_RMCALL_VA_SEM >> 21) & 0xff) == 0,
 	      "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)");
 static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) !=
-	      ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff),
-	      "the GPFIFO and pushbuffer VAs must take distinct small-PT slots");
+	      ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) &&
+	      (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) ==
+	      ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff),
+	      "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104");
 
 /*
  * GspSystemInfo wire ABI, pinned against the reference header compile