gsp-phys: P6b wave 1 - shadow-forward guest CE copies to the physical engine

- the sema service captures the first eligible copy per push from the
  software resolver's own operands (the real push shape mixes guest RAM
  with emulated-VRAM shadow addresses, including physical destinations)
- HPA resolution: guest RAM through the shared guest-RAM file page cache,
  emulated VRAM through the mapping of its kernel shadow
- new scratch page-table pool: 16 zeroed page-table pages with their
  page-directory entries built before any engine access, handing out one
  whole 128-byte GMMU PTE fetch line per copy pair (16 PTEs, 512 pairs
  per boot, never reused; a PTE written into an already-fetched line is
  not observed and there is no invalidate path)
- the backend submits the copy on the module-owned channel and verifies
  the result host-side; every miss falls back to the ladder shadow copy,
  and the software pump remains authoritative
- live (full smoke, both parameters on): req=42 ok=42 verify_ok=42
  verify_fail=0 fallback=0, guest payload roundtrip ALL OK 64/64,
  clean latched release; defaults stay off

4 files changed, 791 insertions, 17 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 765232c..90a0a27 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -52,6 +52,7 @@
 #include "gsp_phys_falcon.h"
 #include "gsp_phys_fwsec.h"
 #include "gsp_phys_gpu.h"
+#include "gsp_sema_service.h"
 #include "gsp_phys_booter_load_img.h"
 #include "gsp_phys_gsp_rm_boot_img.h"
 #include "gsp_phys_registry_img.h"
@@ -337,6 +338,35 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_VA_GUEST_S	(GSP_PHYS_RMCALL_VA_BASE + 0x83000)
 
 /*
+ * P6b wave-1 guest-operand scratch window. The engine's GMMU caches PTE
+ * fetches at 128-byte (16 PTE) granularity with no invalidate path, so a
+ * runtime guest mapping must land on a full 128-byte line the engine has
+ * never fetched. A 64-byte half-line is not enough: two adjacent copy pairs
+ * share one 128-byte fetch line and, once the engine has cached it, the
+ * second pair's PTE writes are invisible (live P6b evidence: the first copy
+ * of a pair verified, every following copy on the same fetch line timed out
+ * with sem=0). This window is a pool of GSP_PHYS_RMCALL_GUEST_PT_PAGES fresh
+ * 2 MB PD0 slots (indices 1..16, right above the ladder's PD0 slot 0) each
+ * backed by its own 4 KB small-page PT; every runtime guest-HPA copy pair
+ * consumes one whole never-used 128-byte line, writing the source PTE at
+ * small-PT slot 16*line and the destination PTE at 16*line+1 within the page
+ * (a line holds GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES = 16 PTEs, so one copy
+ * pair owns the fetch line and the remaining 14 slots are never written).
+ * Lines are allocated monotonically and never reused, so a cached line can
+ * never shadow a later mapping. All PD0 entries are built once in
+ * gsp_phys_rmcall_pdb_build (before any engine access) and every PTE page
+ * starts zeroed (invalid). 16 pages x 32 lines = 512 guest copies per boot
+ * with no reuse.
+ */
+#define GSP_PHYS_RMCALL_VA_GUEST_PT_BASE	0x00200000ULL
+#define GSP_PHYS_RMCALL_GUEST_PT_PAGES		16u
+/* One GMMU PTE fetch line is 128 B = 16 x 8 B PTEs; a pair consumes a line. */
+#define GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES	16u
+/* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */
+#define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE				\
+	((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
+
+/*
  * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
  * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
  * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
@@ -420,6 +450,28 @@ enum gsp_phys_rmcall_buf {
 	GSP_PHYS_RMCALL_BUF_PD1,
 	GSP_PHYS_RMCALL_BUF_PD0,
 	GSP_PHYS_RMCALL_BUF_PT,
+	/*
+	 * P6b wave-1 guest scratch page tables: GSP_PHYS_RMCALL_GUEST_PT_PAGES
+	 * fresh 4 KB PT pages behind PD0 entries 1..16 (never-reused operand
+	 * VA lines). Every proven allocation is a single-page memdesc, so each
+	 * PD0 slot gets its own chunk instead of sharing one multi-page chunk.
+	 */
+	GSP_PHYS_RMCALL_BUF_GUEST_PT0,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT1,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT2,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT3,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT4,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT5,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT6,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT7,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT8,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT9,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT10,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT11,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT12,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT13,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT14,
+	GSP_PHYS_RMCALL_BUF_GUEST_PT15,
 	GSP_PHYS_RMCALL_BUF_COUNT
 };
 
@@ -993,6 +1045,16 @@ static struct gsp_phys_boot_stage {
 	bool rmcall_guest_r_ok;		/* trigger 7: guest page read proved */
 	bool rmcall_guest_w2g_ok;	/* trigger 6: guest page write proved */
 	bool rmcall_guest_selftest_ok;	/* trigger 8: fresh-line PTE write proved */
+	/*
+	 * P6b wave-1 guest forwarding pool. Line index L maps to scratch PT
+	 * page L / LINES_PER_PAGE and 128-byte line-in-page L % LINES_PER_PAGE;
+	 * each line holds one copy pair (src PTE then dst PTE, 14 PTEs spare).
+	 * rmcall_guest_pt_next_line is monotonic for the life of a primed
+	 * staging: a consumed line is never reused, so no engine-cached 128 B
+	 * PTE fetch line can alias a later guest mapping.
+	 */
+	u64 rmcall_va_guest_pt;		/* guest scratch-PT VA window base */
+	u32 rmcall_guest_pt_next_line;	/* next never-used 128 B PTE line */
 	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
 	struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
 	u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
@@ -1545,6 +1607,8 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_guest_r_ok = false;
 	boot_stage.rmcall_guest_w2g_ok = false;
 	boot_stage.rmcall_guest_selftest_ok = false;
+	boot_stage.rmcall_va_guest_pt = 0;
+	boot_stage.rmcall_guest_pt_next_line = 0;
 	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
 		boot_stage.rmcall_mem_handles[i] = 0;
 		boot_stage.rmcall_mem_registered[i] = false;
@@ -5496,7 +5560,12 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0)
 	return 0;
 }
 
-/* One 4 KB coherent chunk per ladder buffer; all freed by rm-call fini. */
+/*
+ * One coherent chunk per ladder buffer; all freed by rm-call fini. Every
+ * buffer is one 4 KB page. The P6b guest scratch pool is
+ * GSP_PHYS_RMCALL_GUEST_PT_PAGES separate single-page chunks (one per PD0
+ * slot), which is the only shape the server accepts for client memdescs.
+ */
 static int gsp_phys_rmcall_alloc_chunks(void)
 {
 	u32 i;
@@ -5511,7 +5580,7 @@ static int gsp_phys_rmcall_alloc_chunks(void)
 			       i);
 			return -EINVAL;
 		}
-		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1)) {
+		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1u)) {
 			pr_err("gsp-phys-boot: rm-call: buffer %u allocation failed\n",
 			       i);
 			return -ENOMEM;
@@ -5552,7 +5621,7 @@ static int gsp_phys_rmcall_stage2(void __iomem *bar0)
 		}
 	}
 
-	pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt zeroed)\n",
+	pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt/guest_pt0..15 zeroed)\n",
 		GSP_PHYS_RMCALL_BUF_COUNT);
 	return 0;
 }
@@ -5783,6 +5852,7 @@ static void gsp_phys_rmcall_pdb_build(void)
 	u32 src_idx = (u32)((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ffu);
 	u32 dst_idx = (u32)((GSP_PHYS_RMCALL_VA_DST >> 12) & 0x1ffu);
 	u32 sem_idx = (u32)((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ffu);
+	u32 i;
 	u64 root_ent, pd2_ent, pd1_ent, pd0_small;
 	u64 pt_gpfifo, pt_pb, pt_src, pt_dst, pt_sem;
 
@@ -5870,6 +5940,29 @@ static void gsp_phys_rmcall_pdb_build(void)
 				boot_stage.rmcall_guest_gpa, gpa_ret);
 		}
 	}
+	/*
+	 * P6b wave-1 scratch PT hierarchy: PD0 entries 1..16 (the 2 MB slots
+	 * right above the ladder window) carry the GSP_PHYS_RMCALL_GUEST_PT_PAGES
+	 * single-page PT chunks (GUEST_PT0..GUEST_PT15). Only the small half of
+	 * each dual entry is used. Writing them here, before any engine access,
+	 * means the first fetch of any 128 B operand line already loads a valid
+	 * hierarchy; the leaf PTEs stay zero (invalid) until the runtime
+	 * allocator consumes a line, and the monotonic allocator never rewrites
+	 * a consumed line.
+	 */
+	for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_PAGES; i++) {
+		u64 guest_pt_pde = gsp_phys_rmcall_pde(
+			(u64)gsp_phys_chunk_page_dma(
+				&boot_stage.rmcall_chunks[
+					GSP_PHYS_RMCALL_BUF_GUEST_PT0 + i], 0));
+
+		put_unaligned_le64(0,
+				   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+				   (size_t)(1 + i) * 16);
+		put_unaligned_le64(guest_pt_pde,
+				   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+				   (size_t)(1 + i) * 16 + 8);
+	}
 	dma_wmb();
 
 	boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE;
@@ -5878,6 +5971,7 @@ static void gsp_phys_rmcall_pdb_build(void)
 	boot_stage.rmcall_va_src = GSP_PHYS_RMCALL_VA_SRC;
 	boot_stage.rmcall_va_dst = GSP_PHYS_RMCALL_VA_DST;
 	boot_stage.rmcall_va_sem = GSP_PHYS_RMCALL_VA_SEM;
+	boot_stage.rmcall_va_guest_pt = GSP_PHYS_RMCALL_VA_GUEST_PT_BASE;
 	boot_stage.rmcall_pdb_root = root_ent;
 	boot_stage.rmcall_pdb_pd2 = pd2_ent;
 	boot_stage.rmcall_pdb_pd1 = pd1_ent;
@@ -5909,6 +6003,11 @@ static void gsp_phys_rmcall_pdb_build(void)
 		src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0),
 		dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0),
 		sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0));
+	pr_info("gsp-phys-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n",
+		GSP_PHYS_RMCALL_GUEST_PT_PAGES,
+		GSP_PHYS_RMCALL_VA_GUEST_PT_BASE,
+		GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+			GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
 }
 
 /*
@@ -6465,6 +6564,246 @@ void gsp_phys_backend_work_counters(u64 *submitted, u64 *released,
 EXPORT_SYMBOL_GPL(gsp_phys_backend_work_counters);
 
 /*
+ * P6b wave-1 guest-operand forwarding API (requires gsp_phys_work_forward=1
+ * AND gsp_phys_guest_mem=1; both default off). It runs one bounded CE copy
+ * on the ladder channel with both operands mapped to real guest RAM by HPA,
+ * consuming one whole fresh, never-reused 128 B PTE fetch line per copy pair
+ * from the guest scratch PT pool: PD0 entries 1..16 -> 16 small-PT pages,
+ * line L = page L / 32, small-PT index 16 * (L % 32) for src and +1 for dst
+ * (a full GMMU fetch line per pair; 64 B half-lines shared by two pairs go
+ * stale because the GMMU caches at 128 B). No guest VA is ever submitted: the
+ * caller passes translated HPAs only.
+ *
+ * Shadow mode: the software pump has already applied the same bytes, so a
+ * successful hardware submission is proven by the translated-HPA submission
+ * being accepted, the module semaphore releasing, and the direct-map verify
+ * comparing source and destination (identity domain + pfn_valid make __va
+ * safe). A late hardware write can theoretically race a guest reuse of the
+ * page; the window is one ordered workqueue hop plus the bounded poll
+ * (<= 200 ms) and wave 2 removes it by making hardware primary.
+ *
+ * Counters: _fwd_ok counts verified copies, _verify_ok/_verify_fail the
+ * direct-map comparison, _pool_exhausted exhausted line allocations and
+ * _refused every attempt that produced no verified copy for another reason
+ * (gate, structural, submit or poll timeout). Plain increments from the
+ * sema service work item; only logging and the debugfs state file read them.
+ */
+static u64 gsp_phys_guest_fwd_ok;
+static u64 gsp_phys_guest_verify_ok;
+static u64 gsp_phys_guest_verify_fail;
+static u64 gsp_phys_guest_pool_exhausted;
+static u64 gsp_phys_guest_refused;
+
+void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail,
+				     u64 *pool_exhausted, u64 *refused)
+{
+	if (ok)
+		*ok = gsp_phys_guest_fwd_ok;
+	if (verify_ok)
+		*verify_ok = gsp_phys_guest_verify_ok;
+	if (verify_fail)
+		*verify_fail = gsp_phys_guest_verify_fail;
+	if (pool_exhausted)
+		*pool_exhausted = gsp_phys_guest_pool_exhausted;
+	if (refused)
+		*refused = gsp_phys_guest_refused;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters);
+
+/*
+ * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool
+ * and write both operand PTEs into it (src at 16*line_in_page, dst at
+ * 16*line_in_page + 1). Called with gsp_phys_rm_lock held, so the monotonic
+ * line index and the PTE writes are serialised. A whole 128 B line is
+ * consumed per copy pair and never reused: the GMMU caches PTE fetches at
+ * 128 B granularity and there is no invalidate, so a later write into an
+ * already-fetched line is invisible - sharing a 128 B line between copy pairs
+ * (one 64 B half each) fails exactly this way in live P6b runs, where only
+ * the first copy of the shared line took effect and every following copy
+ * timed out with sem=0. The scratch PT pages were zeroed at stage 2, so every
+ * unconsumed slot holds an invalid PTE. Returns -ENOSPC when the pool is
+ * exhausted and -EIO when the scratch PT pages are not allocated.
+ */
+static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa,
+					    u64 *src_va, u64 *dst_va)
+{
+	struct gsp_phys_boot_chunk *guest_pt_chunk;
+	u32 line = boot_stage.rmcall_guest_pt_next_line;
+	u32 page, line_in_page;
+	u64 va;
+	u8 *pt;
+
+	if (!boot_stage.rmcall_va_guest_pt)
+		return -EIO;
+	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
+		return -ENOSPC;
+	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
+	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
+	guest_pt_chunk = &boot_stage.rmcall_chunks[
+		GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
+	if (!guest_pt_chunk->pages)
+		return -EIO;
+	/* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */
+	va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
+	     ((u64)line_in_page << 16);
+	pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0);
+	if (!pt)
+		return -EIO;
+	put_unaligned_le64(gsp_phys_rmcall_pte(src_hpa),
+			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+					 line_in_page) * 8);
+	put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa),
+			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+					 line_in_page + 1) * 8);
+	dma_wmb();
+	boot_stage.rmcall_guest_pt_next_line = line + 1;
+
+	*src_va = va;
+	*dst_va = va + GSP_PHYS_PAGE_SIZE;
+	return 0;
+}
+
+int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
+				   u32 dst_off, u32 bytes, u32 poll_ms)
+{
+	struct gsp_phys_boot_chunk *sem =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+	void __iomem *bar0;
+	unsigned long deadline;
+	u64 src_va = 0, dst_va = 0;
+	u32 poll = 0;
+	int ret;
+
+	/*
+	 * Every gate is fail-closed: the sema service's ladder fallback keeps
+	 * the software release authoritative, so a refusal never loses work.
+	 */
+	if (!gsp_phys_backend_work_ready()) {
+		gsp_phys_guest_refused++;
+		return -EAGAIN;
+	}
+	if (!gsp_phys_guest_mem_enabled() ||
+	    gsp_phys_dma_domain_identity() != 1 ||
+	    !gsp_phys_guest_ram_active()) {
+		gsp_phys_guest_refused++;
+		return -EAGAIN;
+	}
+	if (!src_hpa || !dst_hpa ||
+	    !IS_ALIGNED(src_hpa, GSP_PHYS_PAGE_SIZE) ||
+	    !IS_ALIGNED(dst_hpa, GSP_PHYS_PAGE_SIZE) ||
+	    bytes < 4 || bytes > GSP_PHYS_RMCALL_COPY_SIZE || (bytes & 3) ||
+	    (u64)src_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE ||
+	    (u64)dst_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE) {
+		gsp_phys_guest_refused++;
+		pr_err("gsp-phys-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n",
+		       src_hpa, src_off, dst_hpa, dst_off, bytes,
+		       GSP_PHYS_RMCALL_COPY_SIZE);
+		return -EINVAL;
+	}
+	if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS)
+		poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS;
+
+	bar0 = gsp_phys_gpu_bar0();
+	if (!bar0) {
+		gsp_phys_guest_refused++;
+		return -EAGAIN;
+	}
+
+	mutex_lock(&gsp_phys_rm_lock);
+	/*
+	 * Re-check under the lock like the P5 path: the release may have
+	 * committed between the readiness gate and the lock. No device write
+	 * happens on any of these paths.
+	 */
+	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
+	    boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel ||
+	    boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) {
+		mutex_unlock(&gsp_phys_rm_lock);
+		gsp_phys_guest_refused++;
+		return -EAGAIN;
+	}
+
+	ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, &src_va,
+					       &dst_va);
+	if (ret) {
+		if (ret == -ENOSPC) {
+			gsp_phys_guest_pool_exhausted++;
+			if ((gsp_phys_guest_pool_exhausted & 63) == 1)
+				pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n",
+				       gsp_phys_guest_pool_exhausted,
+				       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+				       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+		} else {
+			gsp_phys_guest_refused++;
+			pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n",
+			       ret, src_hpa, dst_hpa);
+		}
+		mutex_unlock(&gsp_phys_rm_lock);
+		return ret;
+	}
+
+	/*
+	 * The scratch PTEs map whole pages, so fold each operand's in-page
+	 * offset into the submitted VAs; the verify below uses the same
+	 * offsets, keeping the transfer and the check consistent.
+	 */
+	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va + src_off,
+					       dst_va + dst_off,
+					       GSP_PHYS_RMCALL_VA_SEM, bytes,
+					       NULL);
+	mutex_unlock(&gsp_phys_rm_lock);
+	if (ret) {
+		gsp_phys_guest_refused++;
+		pr_err("gsp-phys-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
+		       ret, src_hpa, dst_hpa, bytes);
+		return ret;
+	}
+
+	/*
+	 * Poll the module semaphore WITHOUT gsp_phys_rm_lock (same discipline
+	 * as the P5 path): the release is written by the engine, so no host
+	 * path can race it and holding the lock would stall the RPC path.
+	 */
+	deadline = jiffies + msecs_to_jiffies(poll_ms);
+	for (;;) {
+		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
+		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
+			break;
+		if (time_after_eq(jiffies, deadline)) {
+			gsp_phys_guest_refused++;
+			pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
+			       poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD,
+			       src_hpa, dst_hpa, bytes);
+			return -ETIMEDOUT;
+		}
+		msleep(GSP_PHYS_RMCALL_POLL_MS);
+	}
+
+	/*
+	 * Direct-map verify: safe only because the physical device is in an
+	 * identity domain (gated above) and gsp_phys_gpa_to_hpa() validated
+	 * the RAM PFN behind each HPA.
+	 */
+	if (memcmp((u8 *)__va(dst_hpa) + dst_off,
+		   (u8 *)__va(src_hpa) + src_off, bytes) != 0) {
+		gsp_phys_guest_verify_fail++;
+		pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
+		       poll, dst_hpa, dst_off, src_hpa, src_off, bytes);
+		return -EIO;
+	}
+
+	gsp_phys_guest_verify_ok++;
+	gsp_phys_guest_fwd_ok++;
+	if (gsp_phys_guest_fwd_ok == 1 || (gsp_phys_guest_fwd_ok & 63) == 0)
+		pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n",
+			src_hpa, dst_hpa, bytes, poll,
+			gsp_phys_guest_fwd_ok);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
+
+/*
  * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE ->
  * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX ->
  * GET_WORK_SUBMIT_TOKEN - then the first real work item: write the CE method
@@ -7227,6 +7566,36 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rmcall_guest_gpa_valid,
 		   gsp_phys_dma_domain_identity());
 	{
+		/*
+		 * P6b wave-1 forwarding: req/fallback/ok are the sema service's
+		 * request/fallback/verified counters, cand_* and resolve_fail
+		 * its per-request candidate accounting, the rest are the
+		 * booter guest-copy counters and the scratch-PT pool position.
+		 */
+		u64 fwd_req = 0, fwd_fallback = 0;
+		u64 guest_ok = 0, guest_verify_ok = 0, guest_verify_fail = 0;
+		u64 guest_pool = 0, guest_refused = 0;
+		u64 fwd_cand_none = 0, fwd_cand_sys = 0, fwd_cand_mixed = 0;
+		u64 fwd_cand_vid = 0, fwd_resolve_fail = 0;
+
+		gsp_sema_service_fwd_guest_stats(&fwd_req, &fwd_fallback, NULL);
+		gsp_sema_service_fwd_cand_stats(&fwd_cand_none, &fwd_cand_sys,
+						&fwd_cand_mixed, &fwd_cand_vid,
+						&fwd_resolve_fail);
+		gsp_phys_backend_guest_counters(&guest_ok, &guest_verify_ok,
+						&guest_verify_fail,
+						&guest_pool, &guest_refused);
+		seq_printf(m,
+			   "work_guest: req=%llu fallback=%llu ok=%llu verify_ok=%llu verify_fail=%llu pool_exhausted=%llu refused=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu next_line=%u capacity=%u\n",
+			   fwd_req, fwd_fallback, guest_ok, guest_verify_ok,
+			   guest_verify_fail, guest_pool, guest_refused,
+			   fwd_cand_none, fwd_cand_sys, fwd_cand_mixed,
+			   fwd_cand_vid, fwd_resolve_fail,
+			   boot_stage.rmcall_guest_pt_next_line,
+			   GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+				   GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+	}
+	{
 		char guest_ram[512];
 
 		gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram));
@@ -8388,8 +8757,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296
 	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
 	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
-static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 14,
-	      "the rm-call ladder registers exactly fourteen DMA buffers");
+static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 30,
+	      "the rm-call ladder registers exactly thirty DMA buffers (14 ladder + 16 single-page guest scratch PTs)");
 static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
 	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
 	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
@@ -8408,6 +8777,26 @@ static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) !=
 	      (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) ==
 	      ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff),
 	      "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104");
+static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 &&
+	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 &&
+	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 &&
+	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) == 1 &&
+	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) +
+		      GSP_PHYS_RMCALL_GUEST_PT_PAGES <= 256,
+	      "the guest scratch PT window must use PD0 entries 1..16 under the same root/PD2/PD1 chain as the ladder");
+static_assert(GSP_PHYS_RMCALL_VA_GUEST_PT_BASE +
+		      (u64)GSP_PHYS_RMCALL_GUEST_PT_PAGES * (2ULL << 20) <=
+		      GSP_PHYS_RMCALL_VAS_SIZE,
+	      "the guest scratch PT window must stay below the VAS vaSize limit");
+static_assert(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+		      GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE <=
+	      (GSP_PHYS_PAGE_SIZE / 8u),
+	      "each full 128 B guest scratch PT line (16 small-PTEs) must fit "
+	      "one 4 KB PT page");
+static_assert(GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+		      GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE ==
+	      512,
+	      "the guest scratch PT pool must hold 512 one-pair fetch lines (16 pages x 32 lines)");
 
 /*
  * GspSystemInfo wire ABI, pinned against the reference header compile
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index b44d079..cc3b935 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -212,6 +212,36 @@ void gsp_phys_backend_work_counters(u64 *submitted, u64 *released,
 				    u64 *timed_out, u64 *refused);
 
 /*
+ * P6b wave-1 guest-operand forwarding (gsp_phys_work_forward=1 AND
+ * gsp_phys_guest_mem=1; both default off, either off keeps the P5 ladder
+ * shadow exactly as it was).
+ *
+ * gsp_phys_backend_guest_ce_copy() runs one bounded CE copy on the ladder
+ * channel with both operands mapped to real guest RAM by translated host PA:
+ * it re-checks the READY backend, the ladder channel, work forwarding, guest
+ * memory, the identity DMA domain and the active guest-RAM file; consumes one
+ * whole fresh, never-reused 128 B PTE fetch line per copy pair from the guest
+ * scratch PT pool built at stage 3 (no line reuse: the GMMU caches PTE
+ * fetches at 128 B granularity and there is no invalidate, so two pairs
+ * sharing a line would leave the second pair's PTEs stale); then submits
+ * src_off..+bytes from src_hpa to dst_hpa with the module semaphore, polls
+ * it lock-free for up to poll_ms (clamped to 200 ms) and verifies the copy
+ * with a direct-map memcmp (safe: identity domain and pfn_valid HPAs). Returns 0 on a verified copy,
+ * -ENOSPC when the scratch-PT pool is exhausted, -ETIMEDOUT on the poll
+ * bound, -EAGAIN when a gate is closed and -EINVAL/-EIO on operand/structural
+ * failures. No guest VA is ever submitted; the sema service falls back to its
+ * ladder copy on every failure, so the software release never depends on it.
+ *
+ * gsp_phys_backend_guest_counters() reads the booter-side guest counters
+ * (verified copies, direct-map verifies, verify failures, pool exhaustion,
+ * refusals including poll timeouts) for logging and the state file.
+ */
+int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
+				   u32 dst_off, u32 bytes, u32 poll_ms);
+void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail,
+				     u64 *pool_exhausted, u64 *refused);
+
+/*
  * Command-queue TX: build a msgq record with the 570.144 element/rpc header
  * layout, splitting it into 1..16 4 KB elements when it does not fit one
  * (reference elemCount = ceil((48 + rpc.length) / 4096), each element copied
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 25a3cdb..51c51b2 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -42,6 +42,8 @@
 #include <linux/sort.h>
 #include <linux/bitmap.h>
 #include <linux/workqueue.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
 
 #include "nvidia-vgpu-vfio.h"
 #include "gsp_sema_service.h"
@@ -420,6 +422,30 @@ struct sema_svc {
 	u32			doorbell_log;		/* token log budget */
 	bool			doorbell_pending;	/* pump on next service pass */
 
+	/*
+	 * P6b wave-1 guest forwarding (sema-service side): eligible requests
+	 * marked guest, requests that fell back to the ladder copy, verified
+	 * hardware copies. The booter keeps its own submit/verify/pool
+	 * counters; gsp_sema_service_fwd_guest_stats() exposes this triplet
+	 * for the gsp-phys-state work_guest line.
+	 */
+	u64			fwd_guest_req;
+	u64			fwd_guest_fallback;
+	u64			fwd_guest_ok;
+	/*
+	 * P6b wave-1 candidate accounting, one increment per forwarding
+	 * request attempt once the pump is enabled: no eligible candidate was
+	 * captured, or the captured pair's aperture mix. fwd_guest_resolve_fail
+	 * counts guest-work resolution failures that fell back to the ladder
+	 * copy. All zero while the feature is off, so the summary stays silent.
+	 */
+	u64			fwd_cand_none;
+	u64			fwd_cand_sys_sys;
+	u64			fwd_cand_mixed;
+	u64			fwd_cand_vid_vid;
+	u64			fwd_guest_resolve_fail;
+	u64			fwd_guest_resolve_log;	/* first-16 diagnostic budget */
+
 	/* Distinct stalled VA tracking for rate-limited logging. */
 #define SVC_STALLED_MAX		256
 	u64			stalled_vas[SVC_STALLED_MAX];
@@ -1877,6 +1903,19 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
 		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
 		svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop);
+	/*
+	 * P6b wave 1: guest-forwarding summary. Printed whenever any counter
+	 * moved (not only on requests) so a run that captured no candidate is
+	 * still diagnosable from the log.
+	 */
+	if (svc.fwd_guest_req || svc.fwd_cand_none || svc.fwd_cand_sys_sys ||
+	    svc.fwd_cand_mixed || svc.fwd_cand_vid_vid ||
+	    svc.fwd_guest_resolve_fail)
+		pr_info("sema-svc:   work_guest req=%llu ok=%llu fallback=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu\n",
+			svc.fwd_guest_req, svc.fwd_guest_ok,
+			svc.fwd_guest_fallback, svc.fwd_cand_none,
+			svc.fwd_cand_sys_sys, svc.fwd_cand_mixed,
+			svc.fwd_cand_vid_vid, svc.fwd_guest_resolve_fail);
 	/* show a few entries so mappings can be verified from dmesg */
 	if (gsp_sema_dbg) {
 		u32 shown;
@@ -2158,6 +2197,30 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
 	return 0;
 }
 
+/*
+ * P6b wave-1 guest-forwarding candidate: the FIRST linear CE copy in the
+ * decoded push whose operands the software path resolved successfully.
+ * Captured by svc_copy() while the push's work runs, so the candidate is
+ * exactly the copy the software path executed; one capture per decode pass,
+ * consumed by svc_fwd_request() on the same service thread. Each operand
+ * records its aperture kind and the resolver's page/offset: SYS holds a
+ * page-aligned guest GPA, VID holds a page-aligned byte offset into the
+ * emulated BAR1 vidmem shadow. The work item later turns a SYS operand into
+ * an HPA through the guest-RAM backing file and a VID operand into an HPA
+ * through the vmalloc backing of the shadow; no address is ever guessed.
+ * Service-thread only, like the rest of the pump state.
+ */
+struct svc_fwd_cand {
+	bool valid;
+	u8 src_kind;	/* SVC_APERTURE_SYS (GPA) / SVC_APERTURE_FB (shadow) */
+	u8 dst_kind;
+	u64 src_page, dst_page;	/* resolver page: GPA / shadow byte offset */
+	u32 src_off, dst_off;	/* in-page offsets */
+	u32 bytes;
+};
+
+static struct svc_fwd_cand svc_fwd_cand;
+
 /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */
 static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture)
@@ -2200,6 +2263,51 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 	}
 
 	/*
+	 * P6b wave 1: the software path has now resolved both operands, so
+	 * this successful CE copy is the guest-forwarding candidate. Physical
+	 * operands are the norm here (the guest pushes raw vidmem offsets as
+	 * physical destinations, with virtual sources); after the block above,
+	 * a virtual side already holds its resolved pa + aperture, while a
+	 * physical side carries its raw address + the push's aperture, so the
+	 * same page/offset/kind handling covers both. Eligibility stays
+	 * conservative: one page per side, 4-aligned, 1..4096 bytes, and no
+	 * same-page overlap within one aperture; anything doubtful is left to
+	 * the ladder submission. Only the first eligible copy of the push is
+	 * kept (svc_fwd_cand is reset per decode pass).
+	 */
+	if (!svc_fwd_cand.valid &&
+	    total >= 1 && total <= SEMA_PAGE_SIZE && !(total & 3)) {
+		u64 src_page = src & ~(u64)(SEMA_PAGE_SIZE - 1);
+		u64 dst_page = dst & ~(u64)(SEMA_PAGE_SIZE - 1);
+		u32 src_off = (u32)(src & (SEMA_PAGE_SIZE - 1));
+		u32 dst_off = (u32)(dst & (SEMA_PAGE_SIZE - 1));
+		u8 src_kind = (src_aperture == CE_APERTURE_VID) ?
+			SVC_APERTURE_FB : SVC_APERTURE_SYS;
+		u8 dst_kind = (dst_aperture == CE_APERTURE_VID) ?
+			SVC_APERTURE_FB : SVC_APERTURE_SYS;
+		/*
+		 * Same-domain operands inside the same page must not overlap:
+		 * the engine's copy direction would be undefined. Different
+		 * domains (SYS vs VID) are disjoint by construction.
+		 */
+		bool overlap = src_kind == dst_kind && src_page == dst_page &&
+			       src_off < dst_off + total &&
+			       dst_off < src_off + total;
+
+		if (!overlap && src_off + total <= SEMA_PAGE_SIZE &&
+		    dst_off + total <= SEMA_PAGE_SIZE) {
+			svc_fwd_cand.valid = true;
+			svc_fwd_cand.src_kind = src_kind;
+			svc_fwd_cand.dst_kind = dst_kind;
+			svc_fwd_cand.src_page = src_page;
+			svc_fwd_cand.dst_page = dst_page;
+			svc_fwd_cand.src_off = src_off;
+			svc_fwd_cand.dst_off = dst_off;
+			svc_fwd_cand.bytes = (u32)total;
+		}
+	}
+
+	/*
 	 * Data-path watch: for copies of exactly sema_copy_watch bytes, log
 	 * the first 8 source and destination bytes so a broken link in the
 	 * chain (wrong source page, wrong destination, silent wipe) is
@@ -2826,6 +2934,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	}
 
 	memset(&st, 0, sizeof(st));
+	/*
+	 * P6b wave 1: clear the guest-forwarding candidate at the start of
+	 * every decode pass; the first eligible copy captured below is the one
+	 * svc_fwd_request() sees when this pass ends.
+	 */
+	svc_fwd_cand.valid = false;
 	svc.pages_decoded++;
 	if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) {
 		svc_dbg_streams++;
@@ -5020,7 +5134,8 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put)
 }
 
 /* ------------------------------------------------------------------ */
-/* P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1)          */
+/* P5/P6b wave-1 shadow work forwarding                                */
+/* (gsp_phys_work_forward=1, + gsp_phys_guest_mem=1 for guest data)    */
 /* ------------------------------------------------------------------ */
 
 /*
@@ -5029,12 +5144,46 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put)
  * request that arrives while one is in flight (or before the backend/ladder
  * is ready) is dropped and counted. The work item runs the ladder's
  * module-owned copy and observes the real semaphore; the software release
- * path is unchanged and stays authoritative. No guest VA or guest PA ever
- * reaches the engine in this wave.
+ * path is unchanged and stays authoritative.
+ *
+ * P6b wave 1: when svc_copy() captured an eligible linear copy (virtual
+ * operands, one page, resolved by the software path) and gsp_phys_guest_mem=1,
+ * the work item instead resolves each operand to a host PA and runs one real
+ * CE copy against guest memory or the emulated vidmem shadow on a fresh
+ * scratch-PT line: a SYS operand resolves through the guest-RAM backing file,
+ * a VID operand through the vmalloc backing of BAR1. Any failure (translation,
+ * out-of-range shadow, missing vmalloc page, pool exhaustion, submit, timeout,
+ * verify) falls back to the unchanged ladder copy, so the software pump is
+ * never gated by the hardware path. Shadow-mode limit: the software copy
+ * already applied the same bytes, so hardware execution is proven by
+ * acceptance of the translated-HPA submission plus the semaphore release and
+ * the direct-map consistency check; a late hardware write can theoretically
+ * race a guest reuse of the page. The window is one ordered workqueue hop plus
+ * the bounded poll (<= 200 ms); wave 2 removes it by making hardware primary.
  */
 #define SVC_FWD_INFLIGHT	0
 #define SVC_FWD_LOG_MASK	63
 
+/*
+ * One pending request, written by the service thread under svc_fwd_lock
+ * before queue_work and read by the work item without that lock (quiesce
+ * holds that lock across cancel_work_sync, so the work item must never take
+ * it). The SVC_FWD_INFLIGHT bit serialises writers and queue_work orders the
+ * slot write before the work item runs. vgpu is carried for the VID operand
+ * math only; it is stable because the request is single-inflight and quiesce
+ * quiesces the work item before the backend/device state is released.
+ */
+struct svc_fwd_req {
+	bool guest;
+	struct nv_vgpu_device *vgpu;
+	u8 src_kind, dst_kind;	/* SVC_APERTURE_SYS / SVC_APERTURE_FB */
+	u64 src_page, dst_page;	/* SYS: page GPA; VID: shadow byte offset */
+	u32 src_off, dst_off;
+	u32 bytes;
+};
+
+static struct svc_fwd_req svc_fwd_req;
+
 static struct workqueue_struct *svc_fwd_wq;
 static unsigned long svc_fwd_state;
 static u64 svc_fwd_requests;
@@ -5049,14 +5198,129 @@ static u64 svc_fwd_dropped;
 static DEFINE_MUTEX(svc_fwd_lock);
 static bool svc_fwd_paused;
 
+/*
+ * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve
+ * the page-aligned GPA through the guest-RAM backing file; VID operands
+ * resolve the shadow byte offset through the vmalloc backing of BAR1 (the
+ * same buffer the AP helpers use). The resolver's in-page part, if any, is
+ * folded into the offset. Returns 0 on success; -EINVAL for an unknown kind,
+ * -ERANGE when the FB base is missing or the range is out of bounds, -EFAULT
+ * when vmalloc_to_page() fails, and otherwise gsp_phys_gpa_to_hpa()'s errno.
+ * The caller falls back to the ladder copy on any failure.
+ */
+static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind,
+			       u64 pa, u32 off, u32 bytes, u64 *hpa,
+			       u32 *out_off)
+{
+	u64 page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u32 page_off = (u32)(pa & (SEMA_PAGE_SIZE - 1)) + off;
+	int ret;
+
+	if (kind == SVC_APERTURE_FB) {
+		u8 *base = vgpu ? vgpu->bars[1].vaddr : NULL;
+		struct page *pg;
+
+		if (!base || page > vgpu->bars[1].size ||
+		    (u64)page_off + bytes > vgpu->bars[1].size - page)
+			return -ERANGE;
+		pg = vmalloc_to_page(base + page);
+		if (!pg)
+			return -EFAULT;
+		*hpa = (u64)page_to_phys(pg);
+		*out_off = page_off;
+		return 0;
+	}
+	if (kind != SVC_APERTURE_SYS)
+		return -EINVAL;
+	ret = gsp_phys_gpa_to_hpa(page, hpa);
+	if (ret)
+		return ret;
+	*out_off = page_off;
+	return 0;
+}
+
+/* Classify the pass's captured candidate into the forwarding counters. */
+static void svc_fwd_cand_note(void)
+{
+	if (!svc_fwd_cand.valid) {
+		svc.fwd_cand_none++;
+		return;
+	}
+	if (svc_fwd_cand.src_kind == SVC_APERTURE_FB &&
+	    svc_fwd_cand.dst_kind == SVC_APERTURE_FB)
+		svc.fwd_cand_vid_vid++;
+	else if (svc_fwd_cand.src_kind == SVC_APERTURE_SYS &&
+		 svc_fwd_cand.dst_kind == SVC_APERTURE_SYS)
+		svc.fwd_cand_sys_sys++;
+	else
+		svc.fwd_cand_mixed++;
+}
+
 static void svc_fwd_work_fn(struct work_struct *work)
 {
+	bool guest = svc_fwd_req.guest;
 	u64 src_va = 0, dst_va = 0, sem_va = 0;
 	u32 bytes = 0;
 	unsigned long start = jiffies;
 	int ret;
 
 	/*
+	 * P6b wave 1: try the guest hardware copy first when the pump marked
+	 * this request guest. Every failure falls through to the unchanged
+	 * ladder submission below. The request slot is stable here: the
+	 * INFLIGHT bit is held, so only this reader touches it.
+	 */
+	if (guest) {
+		u64 src_hpa = 0, dst_hpa = 0;
+		u32 src_off = 0, dst_off = 0;
+		int src_rc, dst_rc;
+		bool resolved;
+
+		src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
+					     svc_fwd_req.src_kind,
+					     svc_fwd_req.src_page,
+					     svc_fwd_req.src_off,
+					     svc_fwd_req.bytes, &src_hpa,
+					     &src_off);
+		dst_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
+					     svc_fwd_req.dst_kind,
+					     svc_fwd_req.dst_page,
+					     svc_fwd_req.dst_off,
+					     svc_fwd_req.bytes, &dst_hpa,
+					     &dst_off);
+		resolved = !src_rc && !dst_rc;
+		if (resolved &&
+		    !gsp_phys_backend_guest_ce_copy(src_hpa, src_off,
+						    dst_hpa, dst_off,
+						    svc_fwd_req.bytes,
+						    GSP_PHYS_BACKEND_WORK_POLL_MS)) {
+			svc.fwd_guest_ok++;
+			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+			return;
+		}
+		if (!resolved) {
+			svc.fwd_guest_resolve_fail++;
+			svc.fwd_guest_resolve_log++;
+			if (svc.fwd_guest_resolve_log <= 16)
+				pr_info("gsp-phys-work: guest candidate resolve failed: src kind=%u pa=0x%llx off=%u rc=%d dst kind=%u pa=0x%llx off=%u rc=%d vgpu=%d bytes=%u\n",
+					svc_fwd_req.src_kind,
+					svc_fwd_req.src_page,
+					svc_fwd_req.src_off, src_rc,
+					svc_fwd_req.dst_kind,
+					svc_fwd_req.dst_page,
+					svc_fwd_req.dst_off, dst_rc,
+					svc_fwd_req.vgpu ? 1 : 0,
+					svc_fwd_req.bytes);
+		}
+		svc.fwd_guest_fallback++;
+		if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1)
+			pr_info("gsp-phys-work: guest copy fell back to the ladder copy (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n",
+				svc.fwd_guest_req, svc.fwd_guest_ok,
+				svc.fwd_guest_fallback,
+				svc.fwd_guest_resolve_fail);
+	}
+
+	/*
 	 * Re-resolve the ladder operands on the work item: the backend may
 	 * have gone away between the pump's gate and this run. A false return
 	 * means no module buffer is usable; drop without touching anything.
@@ -5086,12 +5350,15 @@ static void svc_fwd_work_fn(struct work_struct *work)
 static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn);
 
 /* Called only from the service thread (the pump). */
-static void svc_fwd_request(const struct svc_channel *ch)
+static void svc_fwd_request(struct nv_vgpu_device *vgpu,
+			    const struct svc_channel *ch)
 {
 	if (!ch->engine_type)
 		return;
 	if (!gsp_phys_work_forward_enabled())
 		return;
+	/* Classify this decode pass's candidate for the run summary. */
+	svc_fwd_cand_note();
 	if (!gsp_phys_backend_work_ready()) {
 		svc_fwd_dropped++;
 		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
@@ -5116,6 +5383,34 @@ static void svc_fwd_request(const struct svc_channel *ch)
 				svc_fwd_dropped);
 		return;
 	}
+	/*
+	 * Fill the request slot before queueing (INFLIGHT is held, so no
+	 * other writer can race it). A captured candidate runs on the guest
+	 * path only when gsp_phys_guest_mem=1; otherwise, and with no
+	 * candidate, the request stays the unchanged ladder shadow.
+	 */
+	svc_fwd_req.guest = svc_fwd_cand.valid &&
+			    gsp_phys_guest_mem_enabled();
+	if (svc_fwd_req.guest) {
+		svc_fwd_req.vgpu = vgpu;
+		svc_fwd_req.src_kind = svc_fwd_cand.src_kind;
+		svc_fwd_req.dst_kind = svc_fwd_cand.dst_kind;
+		svc_fwd_req.src_page = svc_fwd_cand.src_page;
+		svc_fwd_req.dst_page = svc_fwd_cand.dst_page;
+		svc_fwd_req.src_off = svc_fwd_cand.src_off;
+		svc_fwd_req.dst_off = svc_fwd_cand.dst_off;
+		svc_fwd_req.bytes = svc_fwd_cand.bytes;
+		svc.fwd_guest_req++;
+	} else {
+		svc_fwd_req.vgpu = NULL;
+		svc_fwd_req.src_kind = 0;
+		svc_fwd_req.dst_kind = 0;
+		svc_fwd_req.src_page = 0;
+		svc_fwd_req.dst_page = 0;
+		svc_fwd_req.src_off = 0;
+		svc_fwd_req.dst_off = 0;
+		svc_fwd_req.bytes = 0;
+	}
 	svc_fwd_requests++;
 	if (!svc_fwd_wq) {
 		/*
@@ -5158,6 +5453,41 @@ void gsp_sema_service_work_resume(void)
 	mutex_unlock(&svc_fwd_lock);
 }
 
+/*
+ * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest
+ * line (see gsp_sema_service.h). Plain reads: the service thread and the
+ * work item are the only writers and the values are monotonic.
+ */
+void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok)
+{
+	if (req)
+		*req = svc.fwd_guest_req;
+	if (fallback)
+		*fallback = svc.fwd_guest_fallback;
+	if (ok)
+		*ok = svc.fwd_guest_ok;
+}
+
+/*
+ * P6b wave-1 candidate accounting (see gsp_sema_service.h): per-request
+ * classification of the captured operand mix plus guest-work resolution
+ * failures. Same plain-read discipline as the stats above.
+ */
+void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed,
+				     u64 *vid_vid, u64 *resolve_fail)
+{
+	if (none)
+		*none = svc.fwd_cand_none;
+	if (sys_sys)
+		*sys_sys = svc.fwd_cand_sys_sys;
+	if (mixed)
+		*mixed = svc.fwd_cand_mixed;
+	if (vid_vid)
+		*vid_vid = svc.fwd_cand_vid_vid;
+	if (resolve_fail)
+		*resolve_fail = svc.fwd_guest_resolve_fail;
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	/* Only ever called from the service thread. */
@@ -5404,16 +5734,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							ch->op_retry = 0;
 							rel_seen = (svc_rel_count != rel_n_before);
 							/*
-							 * P5 wave 1: shadow this
-							 * CE push on the ladder
-							 * channel. The software
-							 * release above is already
+							 * P5 wave 1 / P6b:
+							 * shadow this CE push
+							 * on the ladder
+							 * channel, or run it
+							 * against guest memory
+							 * when the decode
+							 * captured a trusted
+							 * sysmem copy and
+							 * gsp_phys_guest_mem=1.
+							 * The software release
+							 * above is already
 							 * applied and stays
-							 * authoritative; the work
-							 * item only observes the
-							 * real semaphore.
+							 * authoritative; every
+							 * hardware failure
+							 * falls back to the
+							 * ladder copy.
 							 */
-							svc_fwd_request(ch);
+							svc_fwd_request(vgpu, ch);
 							/*
 							 * A CE push that should carry this
 							 * channel's next tracking release but
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index ae9b7da..9638dde 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -77,6 +77,23 @@ void gsp_sema_service_work_quiesce(void);
 void gsp_sema_service_work_resume(void);
 
 /*
+ * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest
+ * line: guest-eligible requests seen by the pump, requests that fell back to
+ * the ladder copy, and verified hardware copies. Plain reads of the service
+ * counters; safe from the debugfs show path.
+ */
+void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok);
+
+/*
+ * P6b wave-1 candidate accounting for the same work_guest line: how many
+ * forwarding requests found no eligible captured copy, and of those that did,
+ * the operand aperture mix (SYS/SYS, mixed, VID/VID), plus guest-work
+ * resolution failures on the work item. All zero while the feature is off.
+ */
+void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed,
+				     u64 *vid_vid, u64 *resolve_fail);
+
+/*
  * Discover and pin GR (compute) channel rings after the CE region of the
  * channel video-memory allocation. Set false to leave GR rings untranslated
  * (the pump then only runs the CE channels).