gsp-phys: P6b wave 1 - shadow-forward guest CE copies to the physical engine

- the sema service captures the first eligible copy per push from the
  software resolver's own operands (the real push shape mixes guest RAM
  with emulated-VRAM shadow addresses, including physical destinations)
- HPA resolution: guest RAM through the shared guest-RAM file page cache,
  emulated VRAM through the mapping of its kernel shadow
- new scratch page-table pool: 16 zeroed page-table pages with their
  page-directory entries built before any engine access, handing out one
  whole 128-byte GMMU PTE fetch line per copy pair (16 PTEs, 512 pairs
  per boot, never reused; a PTE written into an already-fetched line is
  not observed and there is no invalidate path)
- the backend submits the copy on the module-owned channel and verifies
  the result host-side; every miss falls back to the ladder shadow copy,
  and the software pump remains authoritative
- live (full smoke, both parameters on): req=42 ok=42 verify_ok=42
  verify_fail=0 fallback=0, guest payload roundtrip ALL OK 64/64,
  clean latched release; defaults stay off

1 files changed, 394 insertions, 5 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 765232c..90a0a27 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -52,6 +52,7 @@
 #include "gsp_phys_falcon.h"
 #include "gsp_phys_fwsec.h"
 #include "gsp_phys_gpu.h"
+#include "gsp_sema_service.h"
 #include "gsp_phys_booter_load_img.h"
 #include "gsp_phys_gsp_rm_boot_img.h"
 #include "gsp_phys_registry_img.h"
@@ -337,6 +338,35 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_VA_GUEST_S	(GSP_PHYS_RMCALL_VA_BASE + 0x83000)
 
 /*
+ * P6b wave-1 guest-operand scratch window. The engine's GMMU caches PTE
+ * fetches at 128-byte (16 PTE) granularity with no invalidate path, so a
+ * runtime guest mapping must land on a full 128-byte line the engine has
+ * never fetched. A 64-byte half-line is not enough: two adjacent copy pairs
+ * share one 128-byte fetch line and, once the engine has cached it, the
+ * second pair's PTE writes are invisible (live P6b evidence: the first copy
+ * of a pair verified, every following copy on the same fetch line timed out
+ * with sem=0). This window is a pool of GSP_PHYS_RMCALL_GUEST_PT_PAGES fresh
+ * 2 MB PD0 slots (indices 1..16, right above the ladder's PD0 slot 0) each
+ * backed by its own 4 KB small-page PT; every runtime guest-HPA copy pair
+ * consumes one whole never-used 128-byte line, writing the source PTE at
+ * small-PT slot 16*line and the destination PTE at 16*line+1 within the page
+ * (a line holds GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES = 16 PTEs, so one copy
+ * pair owns the fetch line and the remaining 14 slots are never written).
+ * Lines are allocated monotonically and never reused, so a cached line can
+ * never shadow a later mapping. All PD0 entries are built once in
+ * gsp_phys_rmcall_pdb_build (before any engine access) and every PTE page
+ * starts zeroed (invalid). 16 pages x 32 lines = 512 guest copies per boot
+ * with no reuse.
+ */
+#define GSP_PHYS_RMCALL_VA_GUEST_PT_BASE	0x00200000ULL
+#define GSP_PHYS_RMCALL_GUEST_PT_PAGES		16u
+/* One GMMU PTE fetch line is 128 B = 16 x 8 B PTEs; a pair consumes a line. */
+#define GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES	16u
+/* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */
+#define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE				\
+	((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
+
+/*
  * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
  * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
  * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
@@ -420,6 +450,28 @@ enum gsp_phys_rmcall_buf {
 	GSP_PHYS_RMCALL_BUF_PD1,
 	GSP_PHYS_RMCALL_BUF_PD0,
 	GSP_PHYS_RMCALL_BUF_PT,
+	/*
+	 * P6b wave-1 guest scratch page tables: GSP_PHYS_RMCALL_GUEST_PT_PAGES
+	 * fresh 4 KB PT pages behind PD0 entries 1..16 (never-reused operand
+	 * VA lines). Every proven allocation is a single-page memdesc, so each
+	 * PD0 slot gets its own chunk instead of sharing one multi-page chunk.
+	 */
+	GSP_PHYS_RMCALL_BUF_GUEST_PT0,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT1,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT2,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT3,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT4,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT5,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT6,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT7,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT8,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT9,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT10,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT11,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT12,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT13,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT14,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT15,
 	GSP_PHYS_RMCALL_BUF_COUNT
 };
 
@@ -993,6 +1045,16 @@ static struct gsp_phys_boot_stage {
 	bool rmcall_guest_r_ok;		/* trigger 7: guest page read proved */
 	bool rmcall_guest_w2g_ok;	/* trigger 6: guest page write proved */
 	bool rmcall_guest_selftest_ok;	/* trigger 8: fresh-line PTE write proved */
+	/*
+	 * P6b wave-1 guest forwarding pool. Line index L maps to scratch PT
+	 * page L / LINES_PER_PAGE and 128-byte line-in-page L % LINES_PER_PAGE;
+	 * each line holds one copy pair (src PTE then dst PTE, 14 PTEs spare).
+	 * rmcall_guest_pt_next_line is monotonic for the life of a primed
+	 * staging: a consumed line is never reused, so no engine-cached 128 B
+	 * PTE fetch line can alias a later guest mapping.
+	 */
+	u64 rmcall_va_guest_pt;		/* guest scratch-PT VA window base */
+	u32 rmcall_guest_pt_next_line;	/* next never-used 128 B PTE line */
 	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
 	struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
 	u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
@@ -1545,6 +1607,8 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_guest_r_ok = false;
 	boot_stage.rmcall_guest_w2g_ok = false;
 	boot_stage.rmcall_guest_selftest_ok = false;
+	boot_stage.rmcall_va_guest_pt = 0;
+	boot_stage.rmcall_guest_pt_next_line = 0;
 	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
 		boot_stage.rmcall_mem_handles[i] = 0;
 		boot_stage.rmcall_mem_registered[i] = false;
@@ -5496,7 +5560,12 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0)
 	return 0;
 }
 
-/* One 4 KB coherent chunk per ladder buffer; all freed by rm-call fini. */
+/*
+ * One coherent chunk per ladder buffer; all freed by rm-call fini. Every
+ * buffer is one 4 KB page. The P6b guest scratch pool is
+ * GSP_PHYS_RMCALL_GUEST_PT_PAGES separate single-page chunks (one per PD0
+ * slot), which is the only shape the server accepts for client memdescs.
+ */
 static int gsp_phys_rmcall_alloc_chunks(void)
 {
 	u32 i;
@@ -5511,7 +5580,7 @@ static int gsp_phys_rmcall_alloc_chunks(void)
 			       i);
 			return -EINVAL;
 		}
-		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1)) {
+		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1u)) {
 			pr_err("gsp-phys-boot: rm-call: buffer %u allocation failed\n",
 			       i);
 			return -ENOMEM;
@@ -5552,7 +5621,7 @@ static int gsp_phys_rmcall_stage2(void __iomem *bar0)
 		}
 	}
 
-	pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt zeroed)\n",
+	pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt/guest_pt0..15 zeroed)\n",
 		GSP_PHYS_RMCALL_BUF_COUNT);
 	return 0;
 }
@@ -5783,6 +5852,7 @@ static void gsp_phys_rmcall_pdb_build(void)
 	u32 src_idx = (u32)((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ffu);
 	u32 dst_idx = (u32)((GSP_PHYS_RMCALL_VA_DST >> 12) & 0x1ffu);
 	u32 sem_idx = (u32)((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ffu);
+	u32 i;
 	u64 root_ent, pd2_ent, pd1_ent, pd0_small;
 	u64 pt_gpfifo, pt_pb, pt_src, pt_dst, pt_sem;
 
@@ -5870,6 +5940,29 @@ static void gsp_phys_rmcall_pdb_build(void)
 				boot_stage.rmcall_guest_gpa, gpa_ret);
 		}
 	}
+	/*
+	 * P6b wave-1 scratch PT hierarchy: PD0 entries 1..16 (the 2 MB slots
+	 * right above the ladder window) carry the GSP_PHYS_RMCALL_GUEST_PT_PAGES
+	 * single-page PT chunks (GUEST_PT0..GUEST_PT15). Only the small half of
+	 * each dual entry is used. Writing them here, before any engine access,
+	 * means the first fetch of any 128 B operand line already loads a valid
+	 * hierarchy; the leaf PTEs stay zero (invalid) until the runtime
+	 * allocator consumes a line, and the monotonic allocator never rewrites
+	 * a consumed line.
+	 */
+	for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_PAGES; i++) {
+		u64 guest_pt_pde = gsp_phys_rmcall_pde(
+			(u64)gsp_phys_chunk_page_dma(
+				&boot_stage.rmcall_chunks[
+					GSP_PHYS_RMCALL_BUF_GUEST_PT0 + i], 0));
+
+		put_unaligned_le64(0,
+				   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+				   (size_t)(1 + i) * 16);
+		put_unaligned_le64(guest_pt_pde,
+				   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+				   (size_t)(1 + i) * 16 + 8);
+	}
 	dma_wmb();
 
 	boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE;
@@ -5878,6 +5971,7 @@ static void gsp_phys_rmcall_pdb_build(void)
 	boot_stage.rmcall_va_src = GSP_PHYS_RMCALL_VA_SRC;
 	boot_stage.rmcall_va_dst = GSP_PHYS_RMCALL_VA_DST;
 	boot_stage.rmcall_va_sem = GSP_PHYS_RMCALL_VA_SEM;
+	boot_stage.rmcall_va_guest_pt = GSP_PHYS_RMCALL_VA_GUEST_PT_BASE;
 	boot_stage.rmcall_pdb_root = root_ent;
 	boot_stage.rmcall_pdb_pd2 = pd2_ent;
 	boot_stage.rmcall_pdb_pd1 = pd1_ent;
@@ -5909,6 +6003,11 @@ static void gsp_phys_rmcall_pdb_build(void)
 		src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0),
 		dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0),
 		sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0));
+	pr_info("gsp-phys-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n",
+		GSP_PHYS_RMCALL_GUEST_PT_PAGES,
+		GSP_PHYS_RMCALL_VA_GUEST_PT_BASE,
+		GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+			GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
 }
 
 /*
@@ -6465,6 +6564,246 @@ void gsp_phys_backend_work_counters(u64 *submitted, u64 *released,
 EXPORT_SYMBOL_GPL(gsp_phys_backend_work_counters);
 
 /*
+ * P6b wave-1 guest-operand forwarding API (requires gsp_phys_work_forward=1
+ * AND gsp_phys_guest_mem=1; both default off). It runs one bounded CE copy
+ * on the ladder channel with both operands mapped to real guest RAM by HPA,
+ * consuming one whole fresh, never-reused 128 B PTE fetch line per copy pair
+ * from the guest scratch PT pool: PD0 entries 1..16 -> 16 small-PT pages,
+ * line L = page L / 32, small-PT index 16 * (L % 32) for src and +1 for dst
+ * (a full GMMU fetch line per pair; 64 B half-lines shared by two pairs go
+ * stale because the GMMU caches at 128 B). No guest VA is ever submitted: the
+ * caller passes translated HPAs only.
+ *
+ * Shadow mode: the software pump has already applied the same bytes, so a
+ * successful hardware submission is proven by the translated-HPA submission
+ * being accepted, the module semaphore releasing, and the direct-map verify
+ * comparing source and destination (identity domain + pfn_valid make __va
+ * safe). A late hardware write can theoretically race a guest reuse of the
+ * page; the window is one ordered workqueue hop plus the bounded poll
+ * (<= 200 ms) and wave 2 removes it by making hardware primary.
+ *
+ * Counters: _fwd_ok counts verified copies, _verify_ok/_verify_fail the
+ * direct-map comparison, _pool_exhausted exhausted line allocations and
+ * _refused every attempt that produced no verified copy for another reason
+ * (gate, structural, submit or poll timeout). Plain increments from the
+ * sema service work item; only logging and the debugfs state file read them.
+ */
+static u64 gsp_phys_guest_fwd_ok;
+static u64 gsp_phys_guest_verify_ok;
+static u64 gsp_phys_guest_verify_fail;
+static u64 gsp_phys_guest_pool_exhausted;
+static u64 gsp_phys_guest_refused;
+
+void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail,
+				     u64 *pool_exhausted, u64 *refused)
+{
+	if (ok)
+		*ok = gsp_phys_guest_fwd_ok;
+	if (verify_ok)
+		*verify_ok = gsp_phys_guest_verify_ok;
+	if (verify_fail)
+		*verify_fail = gsp_phys_guest_verify_fail;
+	if (pool_exhausted)
+		*pool_exhausted = gsp_phys_guest_pool_exhausted;
+	if (refused)
+		*refused = gsp_phys_guest_refused;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters);
+
+/*
+ * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool
+ * and write both operand PTEs into it (src at 16*line_in_page, dst at
+ * 16*line_in_page + 1). Called with gsp_phys_rm_lock held, so the monotonic
+ * line index and the PTE writes are serialised. A whole 128 B line is
+ * consumed per copy pair and never reused: the GMMU caches PTE fetches at
+ * 128 B granularity and there is no invalidate, so a later write into an
+ * already-fetched line is invisible - sharing a 128 B line between copy pairs
+ * (one 64 B half each) fails exactly this way in live P6b runs, where only
+ * the first copy of the shared line took effect and every following copy
+ * timed out with sem=0. The scratch PT pages were zeroed at stage 2, so every
+ * unconsumed slot holds an invalid PTE. Returns -ENOSPC when the pool is
+ * exhausted and -EIO when the scratch PT pages are not allocated.
+ */
+static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa,
+					    u64 *src_va, u64 *dst_va)
+{
+	struct gsp_phys_boot_chunk *guest_pt_chunk;
+	u32 line = boot_stage.rmcall_guest_pt_next_line;
+	u32 page, line_in_page;
+	u64 va;
+	u8 *pt;
+
+	if (!boot_stage.rmcall_va_guest_pt)
+		return -EIO;
+	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
+		return -ENOSPC;
+	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
+	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
+	guest_pt_chunk = &boot_stage.rmcall_chunks[
+		GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
+	if (!guest_pt_chunk->pages)
+		return -EIO;
+	/* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */
+	va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
+	     ((u64)line_in_page << 16);
+	pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0);
+	if (!pt)
+		return -EIO;
+	put_unaligned_le64(gsp_phys_rmcall_pte(src_hpa),
+			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+					 line_in_page) * 8);
+	put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa),
+			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+					 line_in_page + 1) * 8);
+	dma_wmb();
+	boot_stage.rmcall_guest_pt_next_line = line + 1;
+
+	*src_va = va;
+	*dst_va = va + GSP_PHYS_PAGE_SIZE;
+	return 0;
+}
+
+int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
+				   u32 dst_off, u32 bytes, u32 poll_ms)
+{
+	struct gsp_phys_boot_chunk *sem =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+	void __iomem *bar0;
+	unsigned long deadline;
+	u64 src_va = 0, dst_va = 0;
+	u32 poll = 0;
+	int ret;
+
+	/*
+	 * Every gate is fail-closed: the sema service's ladder fallback keeps
+	 * the software release authoritative, so a refusal never loses work.
+	 */
+	if (!gsp_phys_backend_work_ready()) {
+		gsp_phys_guest_refused++;
+		return -EAGAIN;
+	}
+	if (!gsp_phys_guest_mem_enabled() ||
+	    gsp_phys_dma_domain_identity() != 1 ||
+	    !gsp_phys_guest_ram_active()) {
+		gsp_phys_guest_refused++;
+		return -EAGAIN;
+	}
+	if (!src_hpa || !dst_hpa ||
+	    !IS_ALIGNED(src_hpa, GSP_PHYS_PAGE_SIZE) ||
+	    !IS_ALIGNED(dst_hpa, GSP_PHYS_PAGE_SIZE) ||
+	    bytes < 4 || bytes > GSP_PHYS_RMCALL_COPY_SIZE || (bytes & 3) ||
+	    (u64)src_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE ||
+	    (u64)dst_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE) {
+		gsp_phys_guest_refused++;
+		pr_err("gsp-phys-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n",
+		       src_hpa, src_off, dst_hpa, dst_off, bytes,
+		       GSP_PHYS_RMCALL_COPY_SIZE);
+		return -EINVAL;
+	}
+	if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS)
+		poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS;
+
+	bar0 = gsp_phys_gpu_bar0();
+	if (!bar0) {
+		gsp_phys_guest_refused++;
+		return -EAGAIN;
+	}
+
+	mutex_lock(&gsp_phys_rm_lock);
+	/*
+	 * Re-check under the lock like the P5 path: the release may have
+	 * committed between the readiness gate and the lock. No device write
+	 * happens on any of these paths.
+	 */
+	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
+	    boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel ||
+	    boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) {
+		mutex_unlock(&gsp_phys_rm_lock);
+		gsp_phys_guest_refused++;
+		return -EAGAIN;
+	}
+
+	ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, &src_va,
+					       &dst_va);
+	if (ret) {
+		if (ret == -ENOSPC) {
+			gsp_phys_guest_pool_exhausted++;
+			if ((gsp_phys_guest_pool_exhausted & 63) == 1)
+				pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n",
+				       gsp_phys_guest_pool_exhausted,
+				       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+				       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+		} else {
+			gsp_phys_guest_refused++;
+			pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n",
+			       ret, src_hpa, dst_hpa);
+		}
+		mutex_unlock(&gsp_phys_rm_lock);
+		return ret;
+	}
+
+	/*
+	 * The scratch PTEs map whole pages, so fold each operand's in-page
+	 * offset into the submitted VAs; the verify below uses the same
+	 * offsets, keeping the transfer and the check consistent.
+	 */
+	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va + src_off,
+					       dst_va + dst_off,
+					       GSP_PHYS_RMCALL_VA_SEM, bytes,
+					       NULL);
+	mutex_unlock(&gsp_phys_rm_lock);
+	if (ret) {
+		gsp_phys_guest_refused++;
+		pr_err("gsp-phys-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
+		       ret, src_hpa, dst_hpa, bytes);
+		return ret;
+	}
+
+	/*
+	 * Poll the module semaphore WITHOUT gsp_phys_rm_lock (same discipline
+	 * as the P5 path): the release is written by the engine, so no host
+	 * path can race it and holding the lock would stall the RPC path.
+	 */
+	deadline = jiffies + msecs_to_jiffies(poll_ms);
+	for (;;) {
+		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
+		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
+			break;
+		if (time_after_eq(jiffies, deadline)) {
+			gsp_phys_guest_refused++;
+			pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
+			       poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD,
+			       src_hpa, dst_hpa, bytes);
+			return -ETIMEDOUT;
+		}
+		msleep(GSP_PHYS_RMCALL_POLL_MS);
+	}
+
+	/*
+	 * Direct-map verify: safe only because the physical device is in an
+	 * identity domain (gated above) and gsp_phys_gpa_to_hpa() validated
+	 * the RAM PFN behind each HPA.
+	 */
+	if (memcmp((u8 *)__va(dst_hpa) + dst_off,
+		   (u8 *)__va(src_hpa) + src_off, bytes) != 0) {
+		gsp_phys_guest_verify_fail++;
+		pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
+		       poll, dst_hpa, dst_off, src_hpa, src_off, bytes);
+		return -EIO;
+	}
+
+	gsp_phys_guest_verify_ok++;
+	gsp_phys_guest_fwd_ok++;
+	if (gsp_phys_guest_fwd_ok == 1 || (gsp_phys_guest_fwd_ok & 63) == 0)
+		pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n",
+			src_hpa, dst_hpa, bytes, poll,
+			gsp_phys_guest_fwd_ok);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
+
+/*
  * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE ->
  * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX ->
  * GET_WORK_SUBMIT_TOKEN - then the first real work item: write the CE method
@@ -7227,6 +7566,36 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rmcall_guest_gpa_valid,
 		   gsp_phys_dma_domain_identity());
 	{
+		/*
+		 * P6b wave-1 forwarding: req/fallback/ok are the sema service's
+		 * request/fallback/verified counters, cand_* and resolve_fail
+		 * its per-request candidate accounting, the rest are the
+		 * booter guest-copy counters and the scratch-PT pool position.
+		 */
+		u64 fwd_req = 0, fwd_fallback = 0;
+		u64 guest_ok = 0, guest_verify_ok = 0, guest_verify_fail = 0;
+		u64 guest_pool = 0, guest_refused = 0;
+		u64 fwd_cand_none = 0, fwd_cand_sys = 0, fwd_cand_mixed = 0;
+		u64 fwd_cand_vid = 0, fwd_resolve_fail = 0;
+
+		gsp_sema_service_fwd_guest_stats(&fwd_req, &fwd_fallback, NULL);
+		gsp_sema_service_fwd_cand_stats(&fwd_cand_none, &fwd_cand_sys,
+						&fwd_cand_mixed, &fwd_cand_vid,
+						&fwd_resolve_fail);
+		gsp_phys_backend_guest_counters(&guest_ok, &guest_verify_ok,
+						&guest_verify_fail,
+						&guest_pool, &guest_refused);
+		seq_printf(m,
+			   "work_guest: req=%llu fallback=%llu ok=%llu verify_ok=%llu verify_fail=%llu pool_exhausted=%llu refused=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu next_line=%u capacity=%u\n",
+			   fwd_req, fwd_fallback, guest_ok, guest_verify_ok,
+			   guest_verify_fail, guest_pool, guest_refused,
+			   fwd_cand_none, fwd_cand_sys, fwd_cand_mixed,
+			   fwd_cand_vid, fwd_resolve_fail,
+			   boot_stage.rmcall_guest_pt_next_line,
+			   GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+				   GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+	}
+	{
 		char guest_ram[512];
 
 		gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram));
@@ -8388,8 +8757,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296
 	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
 	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
-static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 14,
-	      "the rm-call ladder registers exactly fourteen DMA buffers");
+static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 30,
+	      "the rm-call ladder registers exactly thirty DMA buffers (14 ladder + 16 single-page guest scratch PTs)");
 static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
 	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
 	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
@@ -8408,6 +8777,26 @@ static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) !=
 	      (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) ==
 	      ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff),
 	      "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104");
+static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 &&
+	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 &&
+	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 &&
+	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) == 1 &&
+	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) +
+		      GSP_PHYS_RMCALL_GUEST_PT_PAGES <= 256,
+	      "the guest scratch PT window must use PD0 entries 1..16 under the same root/PD2/PD1 chain as the ladder");
+static_assert(GSP_PHYS_RMCALL_VA_GUEST_PT_BASE +
+		      (u64)GSP_PHYS_RMCALL_GUEST_PT_PAGES * (2ULL << 20) <=
+		      GSP_PHYS_RMCALL_VAS_SIZE,
+	      "the guest scratch PT window must stay below the VAS vaSize limit");
+static_assert(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+		      GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE <=
+	      (GSP_PHYS_PAGE_SIZE / 8u),
+	      "each full 128 B guest scratch PT line (16 small-PTEs) must fit "
+	      "one 4 KB PT page");
+static_assert(GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+		      GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE ==
+	      512,
+	      "the guest scratch PT pool must hold 512 one-pair fetch lines (16 pages x 32 lines)");
 
 /*
  * GspSystemInfo wire ABI, pinned against the reference header compile