gsp-phys: P6c C0+C1 - engine-authored guest CE semaphore (default off)

New default-off gsp_phys_relay: for a forwarded guest CE copy whose push
ends in a one-word release, the release page is resolved at forward time,
mapped on a fresh scratch line, and the real engine writes the payload
directly into the guest-visible semaphore page; the deferred-release hold
then verifies that page and skips only that record's software apply. Any
failure (unresolvable target, refused kind, pool exhaustion, poll timeout,
data mismatch, channel teardown) falls back to applying every release in
software, so a release is never lost or duplicated.

- new exported gsp_phys_backend_guest_ce_copy_ex() + relay counters;
  gsp_phys_backend_guest_ce_copy() behavior unchanged (wrapper)
- new work_relay: counters line in gsp-phys-state
- relay limited to CE channels and one-word absolute releases; counter
  and reduction releases stay on the existing paths

Live: param-off regression matched the wave-2 baseline exactly (req=101
ok=101 verify_ok=101, copy_skipped=119, smoke matched=64/64, relay zeros).
With the parameter on: requested=68 ok=67 skipped_sw=67, verify_fail=0,
resolve_fail=0, timeouts=0, smoke 64/64, and the engine-authored guest
semaphore was observed directly (sem=0x00000003 / 0x0000007e at the guest
page). One copy in 68 released its semaphore but failed the host-side data
compare; it fell back to the software apply (fallback accounting
applied+fallback==engaged) with the smoke green - residual to triage, which
is why the parameter stays default-off. A forced-unresolvable run
(requested=103, fallback=105, no relay submissions) kept the smoke green.
Every cycle: 5 power-domain locks held, driver rebound, WPR2 down, no rail
cycles, pstore empty, same boot.

6 files changed, 750 insertions, 75 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 94b0cae..5c36c35 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -6163,10 +6163,13 @@ static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data)
  * (channel_utils.c:812-820). Returns the block size in dwords (13 method
  * pairs = 26 dwords, <= GPFIFO entry LENGTH field, bits 30:10). The ladder
  * stage-4 call passes GSP_PHYS_RMCALL_COPY_SIZE; the P5 shadow submit passes
- * the caller's size (both within one 4 KB ladder page).
+ * the caller's size (both within one 4 KB ladder page). P6c: sem_payload is
+ * the value the release writes; every legacy call site passes
+ * GSP_PHYS_RMCALL_SEM_PAYLOAD so the block is byte-identical to before.
  */
 static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va,
-					 u64 dst_va, u32 bytes)
+					 u64 dst_va, u32 bytes,
+					 u32 sem_payload)
 {
 	u32 *p = pb;
 
@@ -6177,7 +6180,7 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va,
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B,
 				 (u32)sem_va);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD,
-				 GSP_PHYS_RMCALL_SEM_PAYLOAD);
+				 sem_payload);
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER,
 				 (u32)(dst_va >> 32));
 	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER,
@@ -6225,9 +6228,14 @@ struct gsp_phys_rmcall_ce_submit {
  * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the
  * token check. The semaphore is cleared before the doorbell so a stale release
  * from a previous run cannot be observed as this submission's completion.
+ * P6c: sem_payload is the release value the method block carries; legacy
+ * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is
+ * still cleared even when the caller polls a guest page: the clear only
+ * guards the module-sem poll path and is a harmless non-guest write.
  */
 static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
 					    u64 dst_va, u64 sem_va, u32 bytes,
+					    u32 sem_payload,
 					    struct gsp_phys_rmcall_ce_submit *out)
 {
 	struct gsp_phys_boot_chunk *pb =
@@ -6276,7 +6284,7 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
 	wmb();
 
 	words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va,
-					      bytes);
+					      bytes, sem_payload);
 
 	gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
 	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
@@ -6376,7 +6384,9 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va,
 
 	mutex_lock(&gsp_phys_rm_lock);
 	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va,
-					       bytes, &sub);
+					       bytes,
+					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
+					       &sub);
 	mutex_unlock(&gsp_phys_rm_lock);
 	if (ret)
 		return ret;
@@ -6510,7 +6520,9 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va,
 		return -EAGAIN;
 	}
 	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va,
-					       bytes, &sub);
+					       bytes,
+					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
+					       &sub);
 	mutex_unlock(&gsp_phys_rm_lock);
 	if (ret) {
 		gsp_phys_work_refused++;
@@ -6625,6 +6637,39 @@ void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail,
 EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters);
 
 /*
+ * P6c relay-path counters. Only a copy submitted with a nonzero sem_hpa (the
+ * completion relay) moves these; every legacy call keeps moving the guest
+ * counters above only, so with gsp_phys_relay=0 they stay zero and the legacy
+ * lines are identical. Plain increments from the sema service work item; only
+ * logging and the debugfs state file read them.
+ */
+static u64 gsp_phys_relay_requested;
+static u64 gsp_phys_relay_ok;
+static u64 gsp_phys_relay_verify_fail;
+static u64 gsp_phys_relay_timed_out;
+static u64 gsp_phys_relay_refused;
+static u64 gsp_phys_relay_pool_exhausted;
+
+void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
+				     u64 *timeouts, u64 *refused,
+				     u64 *pool_exhausted)
+{
+	if (requested)
+		*requested = gsp_phys_relay_requested;
+	if (ok)
+		*ok = gsp_phys_relay_ok;
+	if (verify_fail)
+		*verify_fail = gsp_phys_relay_verify_fail;
+	if (timeouts)
+		*timeouts = gsp_phys_relay_timed_out;
+	if (refused)
+		*refused = gsp_phys_relay_refused;
+	if (pool_exhausted)
+		*pool_exhausted = gsp_phys_relay_pool_exhausted;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_relay_counters);
+
+/*
  * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool
  * and write both operand PTEs into it (src at 16*line_in_page, dst at
  * 16*line_in_page + 1). Called with gsp_phys_rm_lock held, so the monotonic
@@ -6637,9 +6682,15 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters);
  * timed out with sem=0. The scratch PT pages were zeroed at stage 2, so every
  * unconsumed slot holds an invalid PTE. Returns -ENOSPC when the pool is
  * exhausted and -EIO when the scratch PT pages are not allocated.
+ *
+ * P6c relay: a nonzero sem_hpa additionally writes a third PTE at
+ * 16*line_in_page + 2 and returns sem_va = line_va + 2*PAGE_SIZE; sem_hpa==0
+ * writes nothing and returns sem_va=0. One line per relay copy either way, so
+ * the pool capacity (and its exhaustion behavior) is unchanged.
  */
 static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa,
-					    u64 *src_va, u64 *dst_va)
+					    u64 sem_hpa, u64 *src_va,
+					    u64 *dst_va, u64 *sem_va)
 {
 	struct gsp_phys_boot_chunk *guest_pt_chunk;
 	u32 line = boot_stage.rmcall_guest_pt_next_line;
@@ -6670,11 +6721,21 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa,
 	put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa),
 			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
 					 line_in_page + 1) * 8);
+	/*
+	 * P6c relay: the guest-visible semaphore page gets the third slot of
+	 * the same never-reused line; the write is ordered with the other two
+	 * by the dma_wmb() below, before any doorbell.
+	 */
+	if (sem_hpa)
+		put_unaligned_le64(gsp_phys_rmcall_pte(sem_hpa),
+				   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+						 line_in_page + 2) * 8);
 	dma_wmb();
 	boot_stage.rmcall_guest_pt_next_line = line + 1;
 
 	*src_va = va;
 	*dst_va = va + GSP_PHYS_PAGE_SIZE;
+	*sem_va = sem_hpa ? va + 2 * GSP_PHYS_PAGE_SIZE : 0;
 	return 0;
 }
 
@@ -6688,39 +6749,73 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa,
  * gsp_phys_work_forward requirement, so the P6a triggers can share the same
  * fresh-line copy path with forwarding. *sem_out receives the last
  * semaphore word seen on success or timeout.
+ *
+ * P6c completion relay: a nonzero sem_hpa adds a third PTE on the same fresh
+ * line for the guest-visible semaphore page (pinned with pfn_valid ->
+ * get_page/put_page for the duration), points the CE release methods at
+ * sem_va + sem_off with sem_payload and polls the guest-visible word at
+ * __va(sem_hpa) + sem_off for the exact payload instead of the module
+ * semaphore. A zero sem_hpa keeps the module-semaphore behavior
+ * byte-identical (same gates, same counters, same logs). Relay refusals,
+ * timeouts and verify failures move the relay counters and return an errno
+ * the caller falls back to, so no release can be lost. *out (if given)
+ * always receives the last semaphore word, the elapsed milliseconds and the
+ * return code.
  */
 static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 					 u32 dst_off, u32 bytes, u32 poll_ms,
-					 bool need_work_forward, u32 *sem_out)
+					 bool need_work_forward,
+					 u64 sem_hpa, u32 sem_off,
+					 u32 sem_payload,
+					 struct gsp_phys_backend_copy_result *out,
+					 u32 *sem_out)
 {
 	struct gsp_phys_boot_chunk *sem =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
 	void __iomem *bar0;
+	struct page *sem_page = NULL;
+	unsigned long start = jiffies;
 	unsigned long deadline;
-	u64 src_va = 0, dst_va = 0;
+	u64 src_va = 0, dst_va = 0, sem_va = 0;
 	u32 poll = 0;
+	bool relay = sem_hpa != 0;
 	int ret;
 
+	if (relay)
+		gsp_phys_relay_requested++;
+
 	/*
 	 * Every gate is fail-closed: the sema service's ladder fallback keeps
 	 * the software release authoritative, so a refusal never loses work.
 	 */
 	if (need_work_forward) {
 		if (!gsp_phys_backend_work_ready()) {
-			gsp_phys_guest_refused++;
-			return -EAGAIN;
+			if (relay)
+				gsp_phys_relay_refused++;
+			else
+				gsp_phys_guest_refused++;
+			ret = -EAGAIN;
+			goto out;
 		}
 	} else {
 		if (!gsp_phys_backend_ladder_ready()) {
-			gsp_phys_guest_refused++;
-			return -EAGAIN;
+			if (relay)
+				gsp_phys_relay_refused++;
+			else
+				gsp_phys_guest_refused++;
+			ret = -EAGAIN;
+			goto out;
 		}
 	}
 	if (!gsp_phys_guest_mem_enabled() ||
 	    gsp_phys_dma_domain_identity() != 1 ||
 	    !gsp_phys_guest_ram_active()) {
-		gsp_phys_guest_refused++;
-		return -EAGAIN;
+		if (relay)
+			gsp_phys_relay_refused++;
+		else
+			gsp_phys_guest_refused++;
+		ret = -EAGAIN;
+		goto out;
 	}
 	if (!src_hpa || !dst_hpa ||
 	    !IS_ALIGNED(src_hpa, GSP_PHYS_PAGE_SIZE) ||
@@ -6732,15 +6827,42 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 		pr_err("gsp-phys-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n",
 		       src_hpa, src_off, dst_hpa, dst_off, bytes,
 		       GSP_PHYS_RMCALL_COPY_SIZE);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
+	}
+	if (relay &&
+	    (!IS_ALIGNED(sem_hpa, GSP_PHYS_PAGE_SIZE) ||
+	     (u64)sem_off + 4 > bytes || !sem_payload)) {
+		gsp_phys_relay_refused++;
+		pr_err("gsp-phys-work: GUEST CE RELAY FAILED: invalid semaphore relay sem_hpa=0x%llx sem_off=%u sem_payload=0x%08x bytes=%u (want a page-aligned HPA, sem_off+4 <= copy size, nonzero payload)\n",
+		       sem_hpa, sem_off, sem_payload, bytes);
+		ret = -EINVAL;
+		goto out;
+	}
+	if (relay) {
+		unsigned long pfn = sem_hpa >> PAGE_SHIFT;
+
+		if (!pfn_valid(pfn)) {
+			gsp_phys_relay_refused++;
+			pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore HPA 0x%llx is not a valid RAM page\n",
+			       sem_hpa);
+			ret = -EINVAL;
+			goto out;
+		}
+		sem_page = pfn_to_page(pfn);
+		get_page(sem_page);
 	}
 	if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS)
 		poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS;
 
 	bar0 = gsp_phys_gpu_bar0();
 	if (!bar0) {
-		gsp_phys_guest_refused++;
-		return -EAGAIN;
+		if (relay)
+			gsp_phys_relay_refused++;
+		else
+			gsp_phys_guest_refused++;
+		ret = -EAGAIN;
+		goto out;
 	}
 
 	mutex_lock(&gsp_phys_rm_lock);
@@ -6753,64 +6875,104 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 	    boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel ||
 	    boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) {
 		mutex_unlock(&gsp_phys_rm_lock);
-		gsp_phys_guest_refused++;
-		return -EAGAIN;
+		if (relay)
+			gsp_phys_relay_refused++;
+		else
+			gsp_phys_guest_refused++;
+		ret = -EAGAIN;
+		goto out;
 	}
 
-	ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, &src_va,
-					       &dst_va);
+	ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa,
+					       relay ? sem_hpa : 0, &src_va,
+					       &dst_va, &sem_va);
 	if (ret) {
 		if (ret == -ENOSPC) {
-			gsp_phys_guest_pool_exhausted++;
-			if ((gsp_phys_guest_pool_exhausted & 63) == 1)
-				pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n",
-				       gsp_phys_guest_pool_exhausted,
-				       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
-				       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+			if (relay) {
+				gsp_phys_relay_pool_exhausted++;
+				if ((gsp_phys_relay_pool_exhausted & 63) == 1)
+					pr_err("gsp-phys-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n",
+					       gsp_phys_relay_pool_exhausted,
+					       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+					       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+			} else {
+				gsp_phys_guest_pool_exhausted++;
+				if ((gsp_phys_guest_pool_exhausted & 63) == 1)
+					pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n",
+					       gsp_phys_guest_pool_exhausted,
+					       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+					       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
+			}
 		} else {
-			gsp_phys_guest_refused++;
+			if (relay)
+				gsp_phys_relay_refused++;
+			else
+				gsp_phys_guest_refused++;
 			pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n",
 			       ret, src_hpa, dst_hpa);
 		}
 		mutex_unlock(&gsp_phys_rm_lock);
-		return ret;
+		goto out;
 	}
 
 	/*
 	 * The scratch PTEs map whole pages, so fold each operand's in-page
 	 * offset into the submitted VAs; the verify below uses the same
-	 * offsets, keeping the transfer and the check consistent.
+	 * offsets, keeping the transfer and the check consistent. The relay
+	 * points the release at the guest semaphore PTE at sem_va + sem_off
+	 * and carries the caller's payload.
 	 */
 	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va + src_off,
 					       dst_va + dst_off,
-					       GSP_PHYS_RMCALL_VA_SEM, bytes,
+					       relay ? sem_va + sem_off :
+						       GSP_PHYS_RMCALL_VA_SEM,
+					       bytes,
+					       relay ? sem_payload :
+						       GSP_PHYS_RMCALL_SEM_PAYLOAD,
 					       NULL);
 	mutex_unlock(&gsp_phys_rm_lock);
 	if (ret) {
-		gsp_phys_guest_refused++;
+		if (relay)
+			gsp_phys_relay_refused++;
+		else
+			gsp_phys_guest_refused++;
 		pr_err("gsp-phys-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
 		       ret, src_hpa, dst_hpa, bytes);
-		return ret;
+		goto out;
 	}
 
 	/*
-	 * Poll the module semaphore WITHOUT gsp_phys_rm_lock (same discipline
-	 * as the P5 path): the release is written by the engine, so no host
-	 * path can race it and holding the lock would stall the RPC path.
+	 * Poll the release WITHOUT gsp_phys_rm_lock (same discipline as the
+	 * P5 path): the release is written by the engine, so no host path can
+	 * race it and holding the lock would stall the RPC path. The relay
+	 * watches the guest-visible word through its pin/direct map; the
+	 * legacy path polls the module ladder semaphore exactly as before.
 	 */
 	deadline = jiffies + msecs_to_jiffies(poll_ms);
 	for (;;) {
-		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
-		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
+		if (relay)
+			poll = READ_ONCE(*(u32 *)((u8 *)__va(sem_hpa) + sem_off));
+		else
+			poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
+		if (poll == (relay ? sem_payload : GSP_PHYS_RMCALL_SEM_PAYLOAD))
 			break;
 		if (time_after_eq(jiffies, deadline)) {
-			gsp_phys_guest_refused++;
+			if (relay) {
+				gsp_phys_relay_timed_out++;
+				pr_err("gsp-phys-work: GUEST CE RELAY FAILED: timeout after %u ms (guest sem=0x%08x want=0x%08x) sem_hpa=0x%llx+%u src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
+				       poll_ms, poll, sem_payload, sem_hpa,
+				       sem_off, src_hpa, dst_hpa, bytes);
+			} else {
+				gsp_phys_guest_refused++;
+				pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
+				       poll_ms, poll,
+				       GSP_PHYS_RMCALL_SEM_PAYLOAD,
+				       src_hpa, dst_hpa, bytes);
+			}
 			if (sem_out)
 				*sem_out = poll;
-			pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
-			       poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD,
-			       src_hpa, dst_hpa, bytes);
-			return -ETIMEDOUT;
+			ret = -ETIMEDOUT;
+			goto out;
 		}
 		msleep(GSP_PHYS_RMCALL_POLL_MS);
 	}
@@ -6822,28 +6984,70 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 	 */
 	if (memcmp((u8 *)__va(dst_hpa) + dst_off,
 		   (u8 *)__va(src_hpa) + src_off, bytes) != 0) {
-		gsp_phys_guest_verify_fail++;
-		pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
-		       poll, dst_hpa, dst_off, src_hpa, src_off, bytes);
-		return -EIO;
+		if (relay) {
+			gsp_phys_relay_verify_fail++;
+			pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
+			       poll, dst_hpa, dst_off, src_hpa, src_off,
+			       bytes);
+		} else {
+			gsp_phys_guest_verify_fail++;
+			pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
+			       poll, dst_hpa, dst_off, src_hpa, src_off,
+			       bytes);
+		}
+		ret = -EIO;
+		goto out;
 	}
 
-	gsp_phys_guest_verify_ok++;
-	gsp_phys_guest_fwd_ok++;
 	if (sem_out)
 		*sem_out = poll;
-	if (gsp_phys_guest_fwd_ok == 1 || (gsp_phys_guest_fwd_ok & 63) == 0)
-		pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n",
-			src_hpa, dst_hpa, bytes, poll,
-			gsp_phys_guest_fwd_ok);
-	return 0;
+	if (relay) {
+		gsp_phys_relay_ok++;
+		if (gsp_phys_relay_ok == 1 || (gsp_phys_relay_ok & 63) == 0)
+			pr_info("gsp-phys-work: GUEST CE RELAY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem_hpa=0x%llx+%u sem=0x%08x relays=%llu\n",
+				src_hpa, dst_hpa, bytes, sem_hpa, sem_off,
+				poll, gsp_phys_relay_ok);
+	} else {
+		gsp_phys_guest_verify_ok++;
+		gsp_phys_guest_fwd_ok++;
+		if (gsp_phys_guest_fwd_ok == 1 ||
+		    (gsp_phys_guest_fwd_ok & 63) == 0)
+			pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n",
+				src_hpa, dst_hpa, bytes, poll,
+				gsp_phys_guest_fwd_ok);
+	}
+	ret = 0;
+
+out:
+	if (sem_page)
+		put_page(sem_page);
+	if (out) {
+		out->sem_value = poll;
+		out->elapsed_ms = jiffies_to_msecs(jiffies - start);
+		out->rc = ret;
+	}
+	return ret;
+}
+
+int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa,
+				      u32 dst_off, u32 bytes, u32 poll_ms,
+				      u64 sem_hpa, u32 sem_off,
+				      u32 sem_payload,
+				      struct gsp_phys_backend_copy_result *out)
+{
+	return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off,
+					     bytes, poll_ms, true,
+					     sem_hpa, sem_off, sem_payload,
+					     out, NULL);
 }
+EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy_ex);
 
 int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
 				   u32 dst_off, u32 bytes, u32 poll_ms)
 {
-	return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off,
-					     bytes, poll_ms, true, NULL);
+	return gsp_phys_backend_guest_ce_copy_ex(src_hpa, src_off, dst_hpa,
+						 dst_off, bytes, poll_ms,
+						 0, 0, 0, NULL);
 }
 EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
 
@@ -6861,7 +7065,8 @@ static int gsp_phys_backend_guest_ce_copy_trigger(u64 src_hpa, u32 src_off,
 						  u32 *sem_out)
 {
 	return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off,
-					     bytes, poll_ms, false, sem_out);
+					     bytes, poll_ms, false,
+					     0, 0, 0, NULL, sem_out);
 }
 
 /*
@@ -7043,7 +7248,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC,
 					       GSP_PHYS_RMCALL_VA_DST,
 					       GSP_PHYS_RMCALL_VA_SEM,
-					       GSP_PHYS_RMCALL_COPY_SIZE, &sub);
+					       GSP_PHYS_RMCALL_COPY_SIZE,
+					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
+					       &sub);
 	if (ret) {
 		pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n",
 		       ret);
@@ -7691,6 +7898,29 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 			   gsp_phys_work_hw_primary_enabled());
 	}
 	{
+		/*
+		 * P6c completion relay: the sema service's relay-selection and
+		 * hold-verification counters (requested/ok/fallback/skipped_sw/
+		 * verify_fail/resolve_fail) plus the booter relay copy
+		 * counters (timeouts/pool_exhausted/refused) and the parameter
+		 * state. All zero while the relay is off.
+		 */
+		u64 r_req = 0, r_ok = 0, r_fallback = 0, r_skipped = 0;
+		u64 r_verify_fail = 0, r_resolve_fail = 0;
+		u64 r_timeouts = 0, r_pool = 0, r_refused = 0;
+
+		gsp_sema_service_fwd_relay_stats(&r_req, &r_ok, &r_fallback,
+						 &r_skipped, &r_verify_fail,
+						 &r_resolve_fail);
+		gsp_phys_backend_relay_counters(NULL, NULL, NULL, &r_timeouts,
+						&r_refused, &r_pool);
+		seq_printf(m,
+			   "work_relay: requested=%llu ok=%llu fallback=%llu skipped_sw=%llu verify_fail=%llu resolve_fail=%llu timeouts=%llu pool_exhausted=%llu refused=%llu relay_param=%u\n",
+			   r_req, r_ok, r_fallback, r_skipped, r_verify_fail,
+			   r_resolve_fail, r_timeouts, r_pool, r_refused,
+			   gsp_phys_relay_enabled());
+	}
+	{
 		char guest_ram[512];
 
 		gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram));
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index ada3280..408d380 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -235,12 +235,48 @@ void gsp_phys_backend_work_counters(u64 *submitted, u64 *released,
  * gsp_phys_backend_guest_counters() reads the booter-side guest counters
  * (verified copies, direct-map verifies, verify failures, pool exhaustion,
  * refusals including poll timeouts) for logging and the state file.
+ *
+ * P6c completion relay (gsp_phys_relay=1; default off):
+ * gsp_phys_backend_guest_ce_copy_ex() is the same copy with an optional
+ * guest-visible release relay: a nonzero sem_hpa (page aligned) makes the
+ * submission map that page as a third PTE on the same fresh 128 B line and
+ * point the CE release methods at sem_va + sem_off with sem_payload; the
+ * caller's page is pinned for the duration (pfn_valid -> get_page/put_page)
+ * and the poll watches __va(sem_hpa) + sem_off for the exact payload instead
+ * of the module ladder semaphore. *out (if given) always carries the last
+ * semaphore word observed, the elapsed milliseconds and the return code. A
+ * zero sem_hpa keeps the module-semaphore behavior byte-identical.
+ * gsp_phys_backend_guest_ce_copy() is a wrapper for the zero-relay form.
  */
 int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
 				   u32 dst_off, u32 bytes, u32 poll_ms);
 void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail,
 				     u64 *pool_exhausted, u64 *refused);
 
+/* Result of one guest CE copy (see the P6c note above). */
+struct gsp_phys_backend_copy_result {
+	u32 sem_value;		/* last word observed at the polled semaphore */
+	u32 elapsed_ms;		/* return-to-caller elapsed milliseconds */
+	int rc;			/* duplicated function return code */
+};
+
+int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa,
+				      u32 dst_off, u32 bytes, u32 poll_ms,
+				      u64 sem_hpa, u32 sem_off,
+				      u32 sem_payload,
+				      struct gsp_phys_backend_copy_result *out);
+
+/*
+ * gsp_phys_backend_relay_counters() reads the P6c relay-path counters
+ * (requested relay copies, verified relays, data verify failures, guest
+ * semaphore poll timeouts, gate/structural refusals and scratch-pool
+ * exhaustion) for logging and the gsp-phys-state work_relay line. All zero
+ * unless gsp_phys_relay=1 copies ran.
+ */
+void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
+				     u64 *timeouts, u64 *refused,
+				     u64 *pool_exhausted);
+
 /*
  * Command-queue TX: build a msgq record with the 570.144 element/rpc header
  * layout, splitting it into 1..16 4 KB elements when it does not fit one
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 512d30b..fc84c1d 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -248,6 +248,31 @@ bool gsp_phys_work_hw_primary_enabled(void)
 EXPORT_SYMBOL_GPL(gsp_phys_work_hw_primary_enabled);
 
 /*
+ * P6c completion relay (default 0). When set together with
+ * gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and
+ * gsp_phys_work_defer_release=1, a forwarded guest CE copy additionally maps
+ * the guest-visible semaphore page into the CE channel page tables and lets
+ * the engine author one selected one-word release into it (custom payload).
+ * The hold then verifies the guest word carries the payload and skips the
+ * software apply of that one record; every refusal, hardware failure,
+ * timeout, abort or failed verification leaves the unchanged software apply
+ * in charge, so the release stays exactly-once. With 0, every behavior is
+ * byte-identical to wave 2 (the module semaphore is polled; no guest
+ * semaphore page is ever mapped or pinned). One-line rollback:
+ * gsp_phys_relay=0.
+ */
+static bool gsp_phys_relay_param;
+module_param_named(gsp_phys_relay, gsp_phys_relay_param, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_relay,
+		 "Let the forwarded guest CE copy author the selected guest-visible semaphore release into the guest semaphore page (default 0: software release stays the sole author; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)");
+
+bool gsp_phys_relay_enabled(void)
+{
+	return gsp_phys_relay_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_relay_enabled);
+
+/*
  * One-way power latch (default). Re-enabling runtime PM and D3cold after a
  * claim lets the board power-gate and the host hard-freeze on resume
  * (observed 2026-10-01); keep the pre-claim D0 policy instead and require
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 70b98e4..6bfe3f0 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -124,6 +124,21 @@ bool gsp_phys_work_defer_release_enabled(void);
 bool gsp_phys_work_hw_primary_enabled(void);
 
 /*
+ * True when the gsp_phys_relay module parameter is set (default 0: a
+ * forwarded guest CE copy polls the module-owned ladder semaphore and the
+ * software release path stays the sole author of the guest-visible release).
+ * When set with gsp_phys_work_forward, gsp_phys_guest_mem and
+ * gsp_phys_work_defer_release, one selected one-word release of a forwarded
+ * guest push is authored by the real CE engine directly into the
+ * guest-visible semaphore page (a third scratch PTE maps that page) and the
+ * service thread skips the software apply of that record once the guest word
+ * carries the payload; every failure, timeout, abort or failed verification
+ * falls back to the unchanged software apply, so a release is never lost or
+ * duplicated. One-line rollback: gsp_phys_relay=0.
+ */
+bool gsp_phys_relay_enabled(void);
+
+/*
  * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder
  * support). The mdev is registered with vfio_register_emulated_iommu_dev():
  * its VFIO type-1 container keeps the guest DMA mappings privately, so
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 1f599fc..dea8249 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -470,6 +470,22 @@ struct sema_svc {
 	u64			fwd_hold_engaged;
 	u64			fwd_hold_timeouts;
 	u64			fwd_hold_canceled;
+	/*
+	 * P6c completion relay (gsp_phys_relay=1 only): relay selections armed
+	 * on a deferred guest request (fwd_relay_requested), relays whose
+	 * engine-authored guest semaphore was verified with the software apply
+	 * skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to
+	 * the software apply (fwd_relay_fallback), hold re-reads that did not
+	 * see the payload (fwd_relay_verify_fail) and work-item semaphore HPA
+	 * resolution failures (fwd_relay_resolve_fail). All zero while the
+	 * parameter is off.
+	 */
+	u64			fwd_relay_requested;
+	u64			fwd_relay_ok;
+	u64			fwd_relay_fallback;
+	u64			fwd_relay_skipped_sw;
+	u64			fwd_relay_verify_fail;
+	u64			fwd_relay_resolve_fail;
 
 	/* Distinct stalled VA tracking for rate-limited logging. */
 #define SVC_STALLED_MAX		256
@@ -2308,6 +2324,18 @@ struct svc_fwd_rel {
 	u8	kind;		/* SVC_FWD_REL_* */
 	u8	op;		/* 906f: host sem op / launch: pending flag */
 	u8	inc_claimed;	/* launch: one-shot inc release claimed */
+	/*
+	 * P6c completion relay: the release target resolved once at record
+	 * time by svc_fwd_relay_note() (only while gsp_phys_relay is on and
+	 * every relay guard passes). tgt_page is a page-aligned GPA (SYS) or
+	 * BAR1 shadow byte offset (FB); tgt_off is the in-page offset; a
+	 * false tgt_ok means the record is software-applied and never
+	 * relayed.
+	 */
+	u8	tgt_ok;
+	u8	tgt_kind;	/* SVC_APERTURE_SYS / SVC_APERTURE_FB */
+	u64	tgt_page;
+	u32	tgt_off;
 	u64	va;
 	u32	payload;
 	u32	entry_idx;
@@ -3133,6 +3161,67 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
 	return false;
 }
 
+/*
+ * P6c completion relay: resolve one deferred release record's target once at
+ * record time so the request can hand the engine a page/offset it may
+ * program. Fail-open by construction: every miss (parameter off, non-CE
+ * channel, GR counter VA, a non-one-word or reduction LAUNCH release, an
+ * unresolvable VA, an unreadable word, or a word already at/ahead of the
+ * payload) leaves tgt_ok false and the record is software-applied exactly as
+ * in wave 1.5. The resolution uses the same svc_resolve_va() the software
+ * apply path uses and pins the page via svc_pin_store(), so the work item's
+ * later svc_fwd_operand_hpa() resolves the same page. Service-thread only.
+ */
+static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu,
+			       const struct svc_channel *ch,
+			       struct svc_fwd_rel *r, u32 launch_dma,
+			       bool is_launch)
+{
+	u64 addr = 0;
+	u32 ap = 0;
+	u32 cur = 0;
+
+	r->tgt_ok = false;
+	r->tgt_kind = 0;
+	r->tgt_page = 0;
+	r->tgt_off = 0;
+
+	if (!gsp_phys_relay_enabled() || !ch || !ch->engine_type)
+		return;
+	if (svc_counter_va(r->va))
+		return;
+	/*
+	 * The relay always authors an absolute one-word release (the CE block
+	 * writes SET_SEMAPHORE_PAYLOAD and LAUNCH_DMA RELEASE_ONE_WORD), so a
+	 * record whose software apply is additive must never be relayed: the
+	 * engine would leave the guest word short of the value it waits on.
+	 * The LAUNCH increment is excluded by the dma check below; the 906F
+	 * REDUCTION op is excluded here.
+	 */
+	if (!is_launch && r->kind == SVC_FWD_REL_906F &&
+	    r->op == HOST_SEM_OP_REDUCTION)
+		return;
+	if (is_launch) {
+		if (DMA_SEMAPHORE_TYPE(launch_dma) !=
+		    SEM_RELEASE_ONE_WORD)
+			return;
+		if (DMA_SEM_REDUCTION_EN(launch_dma) &&
+		    DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC)
+			return;
+	}
+	if (svc_resolve_va(vgpu, r->va, &addr, &ap))
+		return;
+	if (svc_sem_read(vgpu, r->va, &cur))
+		return;
+	/* Forward-only pre-check: only a word behind the payload is relayed. */
+	if (cur >= r->payload)
+		return;
+	r->tgt_ok = true;
+	r->tgt_kind = ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
+	r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1);
+	r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1));
+}
+
 static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			      u32 len, u64 page_pa, bool allow_ops,
 			      struct svc_channel *ch, u32 entry_idx,
@@ -3399,6 +3488,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						r->payload = payload;
 						r->entry_idx = entry_idx;
 						r->dma = 0;
+						svc_fwd_relay_note(vgpu, ch, r,
+								   0, false);
 						svc_fwd_defer_armed = true;
 						svc.fwd_rel_deferred++;
 					} else {
@@ -3512,6 +3603,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 							r->payload = st.rs_payload;
 							r->entry_idx = entry_idx;
 							r->dma = 0;
+							svc_fwd_relay_note(vgpu, ch,
+									   r, 0,
+									   false);
 							svc_fwd_defer_armed = true;
 							svc.fwd_rel_deferred++;
 							break;
@@ -3844,6 +3938,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						r->payload = st.sem_payload;
 						r->entry_idx = entry_idx;
 						r->dma = dma;
+						svc_fwd_relay_note(vgpu, ch, r,
+								   dma, true);
 						/*
 						 * P6b wave 1.5: do NOT bump
 						 * svc.pushes_pending here -
@@ -5486,6 +5582,25 @@ struct svc_fwd_req {
 	u64 src_page, dst_page;	/* SYS: page GPA; VID: shadow byte offset */
 	u32 src_off, dst_off;
 	u32 bytes;
+	/*
+	 * P6c completion relay: when relay is true this request's hardware
+	 * copy additionally authors the guest-visible release of the selected
+	 * deferred record (relay_index into svc_fwd_rel[]): relay_kind/page/
+	 * off identify the resolved semaphore target and relay_payload is the
+	 * value the engine writes. relay_done/relay_sem/relay_elapsed_ms are
+	 * published by the work item before its done release, for the hold's
+	 * verification/logging. Cleared with the rest of the slot whenever the
+	 * releases do not qualify.
+	 */
+	bool relay;
+	u8 relay_index;
+	u8 relay_kind;
+	u64 relay_page;
+	u32 relay_off;
+	u32 relay_payload;
+	bool relay_done;
+	u32 relay_sem;
+	u32 relay_elapsed_ms;
 };
 
 static struct svc_fwd_req svc_fwd_req;
@@ -5669,6 +5784,15 @@ struct svc_fwd_hold {
 	u64 gpfifo_va;
 	u32 entry_idx;
 	u8 nrel;
+	/*
+	 * P6c completion relay: the request selected record relay_index and
+	 * the hardware copy was asked to author it into the guest-visible
+	 * semaphore page. On the success edge the hold re-reads the guest
+	 * word and, when it carries the payload, skips that one record's
+	 * software apply. Cleared with the hold.
+	 */
+	bool relay_armed;
+	u8 relay_index;
 	struct svc_fwd_rel rel[SVC_FWD_REL_MAX];
 	u64 last_rel_va;
 	unsigned long since;
@@ -5733,14 +5857,26 @@ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu,
 			     inc, r->inc_claimed);
 }
 
-/* Apply the recorded releases of one deferred push (service thread). */
-static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu,
-				  const struct svc_channel *ch,
-				  const struct svc_fwd_rel *rel, u8 n)
+/* Skip value that never matches a record index. */
+#define SVC_FWD_REL_SKIP_NONE	0xff
+
+/*
+ * Apply the recorded releases of one deferred push (service thread), except
+ * the record at `skip` (SVC_FWD_REL_SKIP_NONE applies all). P6c uses the skip
+ * on the relay verification success edge: the engine already authored that
+ * one release into the guest-visible semaphore, so re-applying it in software
+ * is unnecessary (and would count a second release).
+ */
+static void svc_fwd_rel_apply_all_ex(struct nv_vgpu_device *vgpu,
+				     const struct svc_channel *ch,
+				     const struct svc_fwd_rel *rel, u8 n,
+				     u8 skip)
 {
 	u8 i;
 
 	for (i = 0; i < n; i++) {
+		if (i == skip)
+			continue;
 		switch (rel[i].kind) {
 		case SVC_FWD_REL_906F:
 			svc_rel_apply_906f(vgpu, ch, rel[i].va,
@@ -5759,6 +5895,14 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu,
 	}
 }
 
+/* Apply every recorded release (no relay skip). */
+static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu,
+				  const struct svc_channel *ch,
+				  const struct svc_fwd_rel *rel, u8 n)
+{
+	svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE);
+}
+
 static bool svc_fwd_hold_expired(void)
 {
 	return time_after(jiffies, svc_fwd_hold.since +
@@ -5790,6 +5934,8 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu,
 	svc_fwd_hold.nrel = n;
 	svc_fwd_hold.last_rel_va = svc_rel_va_seen;
 	svc_fwd_hold.since = jiffies;
+	svc_fwd_hold.relay_armed = svc_fwd_req.relay;
+	svc_fwd_hold.relay_index = svc_fwd_req.relay_index;
 	svc.fwd_hold_engaged++;
 	svc.fwd_rel_pending = n;
 }
@@ -5813,6 +5959,8 @@ static void svc_fwd_hold_clear(void)
 	memset(&svc_fwd_hold.cand, 0, sizeof(svc_fwd_hold.cand));
 	svc_fwd_hold.vgpu = NULL;
 	svc_fwd_hold.nrel = 0;
+	svc_fwd_hold.relay_armed = false;
+	svc_fwd_hold.relay_index = 0;
 	svc.fwd_rel_pending = 0;
 }
 
@@ -5861,6 +6009,13 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu,
 		if (!done)
 			svc.fwd_hold_timeouts++;
 	}
+	/*
+	 * P6b/P6c: the timeout path never reaches the relay verification
+	 * edge, so a relayed set is software-applied whole (forward-only, so
+	 * the engine's write cannot be duplicated).
+	 */
+	if (svc_fwd_hold.relay_armed)
+		svc.fwd_relay_fallback++;
 	svc_fwd_hold_clear();
 }
 
@@ -5883,16 +6038,63 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
 	if (!done && !expired && !aborted)
 		return false;
 	if (done && svc_fwd_req.hw_rc == 0) {
-		svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel,
-				      svc_fwd_hold.nrel);
-		svc.fwd_rel_applied++;
+		/*
+		 * P6c: on the relay success edge, re-read the guest-visible
+		 * semaphore through the software view. Only a word that
+		 * actually carries the payload lets the one relayed record be
+		 * skipped; anything else (unreadable, behind) is a relay
+		 * verification failure and the software applies the whole set
+		 * (the forward-only apply makes the relayed write idempotent,
+		 * so the release is never duplicated or lost).
+		 */
+		if (svc_fwd_hold.relay_armed &&
+		    svc_fwd_hold.relay_index < svc_fwd_hold.nrel) {
+			struct svc_fwd_rel *rr =
+				&svc_fwd_hold.rel[svc_fwd_hold.relay_index];
+			u32 cur = 0;
+
+			if (!svc_sem_read(hv, rr->va, &cur) &&
+			    cur >= rr->payload) {
+				svc_fwd_rel_apply_all_ex(hv, ch,
+							 svc_fwd_hold.rel,
+							 svc_fwd_hold.nrel,
+							 svc_fwd_hold.relay_index);
+				svc.fwd_relay_ok++;
+				svc.fwd_relay_skipped_sw++;
+				svc.fwd_rel_applied++;
+				/*
+				 * The hold's post is the single completion
+				 * edge for this channel: the pump's per-pass
+				 * notifier was suppressed while the relay hold
+				 * was active.
+				 */
+				svc_notify_complete(hv, ch);
+			} else {
+				svc.fwd_relay_verify_fail++;
+				svc.fwd_relay_fallback++;
+				svc_fwd_rel_apply_all(hv, ch,
+						      svc_fwd_hold.rel,
+						      svc_fwd_hold.nrel);
+				svc.fwd_rel_applied++;
+				if ((svc.fwd_relay_verify_fail & 15) == 1)
+					pr_err("gsp-phys-work: RELAY VERIFY FAILED: guest sem va=0x%llx cur=0x%08x payload=0x%08x; software applied all releases\n",
+					       rr->va, cur, rr->payload);
+			}
+		} else {
+			svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel,
+					      svc_fwd_hold.nrel);
+			svc.fwd_rel_applied++;
+		}
 	} else {
 		/*
 		 * P6b wave 2: failure/timeout/abort fallback - author the
 		 * skipped data once, then apply the releases exactly as
-		 * wave 1.5 does.
+		 * wave 1.5 does. P6c: a relayed set is applied whole here
+		 * (the engine did not deliver it).
 		 */
 		svc_fwd_hold_soft_copy(hv);
+		if (svc_fwd_hold.relay_armed)
+			svc.fwd_relay_fallback++;
 		svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 		svc.fwd_rel_fallback++;
@@ -5973,6 +6175,13 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu)
 		svc_fwd_hold_soft_copy(hv);
 		svc.fwd_hold_canceled++;
 	}
+	/*
+	 * P6c: a vanished channel cannot run the relay verification edge; a
+	 * relayed set falls back to the software path (which is exactly-once
+	 * against the engine's forward write).
+	 */
+	if (svc_fwd_hold.relay_armed)
+		svc.fwd_relay_fallback++;
 	svc_fwd_hold_clear();
 }
 
@@ -6006,6 +6215,8 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
 		svc.fwd_rel_applied++;
 	else
 		svc.fwd_rel_fallback++;
+	if (svc_fwd_hold.relay_armed)
+		svc.fwd_relay_fallback++;
 	svc_fwd_hold_clear();
 }
 
@@ -6029,9 +6240,10 @@ static void svc_fwd_work_fn(struct work_struct *work)
 	 * while a hold is active.
 	 */
 	if (guest) {
-		u64 src_hpa = 0, dst_hpa = 0;
-		u32 src_off = 0, dst_off = 0;
-		int src_rc, dst_rc;
+		u64 src_hpa = 0, dst_hpa = 0, sem_hpa = 0;
+		u32 src_off = 0, dst_off = 0, sem_off = 0;
+		struct gsp_phys_backend_copy_result cp = { 0 };
+		int src_rc, dst_rc, sem_rc = 0;
 		bool resolved;
 
 		src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
@@ -6047,11 +6259,43 @@ static void svc_fwd_work_fn(struct work_struct *work)
 					     svc_fwd_req.bytes, &dst_hpa,
 					     &dst_off);
 		resolved = !src_rc && !dst_rc;
+		/*
+		 * P6c: resolve the selected relay target to an HPA with the
+		 * same operand resolver. A failure is not fatal - hw_rc is
+		 * forced to an error so the hold applies the software
+		 * release exactly as wave 1.5 (never lost, never duplicated).
+		 */
+		if (svc_fwd_req.relay)
+			sem_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
+						     svc_fwd_req.relay_kind,
+						     svc_fwd_req.relay_page,
+						     svc_fwd_req.relay_off, 4,
+						     &sem_hpa, &sem_off);
 		if (resolved) {
-			hw_rc = gsp_phys_backend_guest_ce_copy(src_hpa, src_off,
-							       dst_hpa, dst_off,
-							       svc_fwd_req.bytes,
-							       GSP_PHYS_BACKEND_WORK_POLL_MS);
+			if (svc_fwd_req.relay && !sem_rc) {
+				hw_rc = gsp_phys_backend_guest_ce_copy_ex(
+					src_hpa, src_off, dst_hpa, dst_off,
+					svc_fwd_req.bytes,
+					GSP_PHYS_BACKEND_WORK_POLL_MS,
+					sem_hpa, sem_off,
+					svc_fwd_req.relay_payload, &cp);
+				svc_fwd_req.relay_sem = cp.sem_value;
+				svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms;
+				svc_fwd_req.relay_done = (hw_rc == 0);
+			} else if (svc_fwd_req.relay) {
+				hw_rc = -EINVAL;
+				svc.fwd_relay_resolve_fail++;
+				if (svc.fwd_relay_resolve_fail <= 16)
+					pr_info("gsp-phys-work: relay semaphore resolve failed: kind=%u page=0x%llx off=%u rc=%d; software release stays authoritative\n",
+						svc_fwd_req.relay_kind,
+						svc_fwd_req.relay_page,
+						svc_fwd_req.relay_off, sem_rc);
+			} else {
+				hw_rc = gsp_phys_backend_guest_ce_copy(
+					src_hpa, src_off, dst_hpa, dst_off,
+					svc_fwd_req.bytes,
+					GSP_PHYS_BACKEND_WORK_POLL_MS);
+			}
 			if (!hw_rc) {
 				svc.fwd_guest_ok++;
 				if (!svc_fwd_req.defer) {
@@ -6145,6 +6389,56 @@ static void svc_fwd_work_fn(struct work_struct *work)
 static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn);
 
 /*
+ * P6c completion relay: pick the ONE deferred release record this request's
+ * hardware copy may author. Only a deferred guest request whose records were
+ * armed can relay, and only a record whose record-time resolution succeeded
+ * (tgt_ok) qualifies; the push-ending LAUNCH record is preferred (it is the
+ * completion edge the guest tracker waits on), else the last eligible record.
+ * Fail-open: no eligible record leaves req->relay false and the whole release
+ * set stays software-applied. Service-thread only; called with the request
+ * slot's INFLIGHT bit held, before queue_work.
+ */
+static void svc_fwd_relay_select(struct svc_fwd_req *req)
+{
+	int i;
+
+	req->relay = false;
+	req->relay_index = 0;
+	req->relay_kind = 0;
+	req->relay_page = 0;
+	req->relay_off = 0;
+	req->relay_payload = 0;
+
+	if (!req->defer || !gsp_phys_relay_enabled())
+		return;
+	for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) {
+		if (svc_fwd_rel[i].kind == SVC_FWD_REL_LAUNCH &&
+		    svc_fwd_rel[i].tgt_ok) {
+			req->relay = true;
+			req->relay_index = (u8)i;
+			req->relay_kind = svc_fwd_rel[i].tgt_kind;
+			req->relay_page = svc_fwd_rel[i].tgt_page;
+			req->relay_off = svc_fwd_rel[i].tgt_off;
+			req->relay_payload = svc_fwd_rel[i].payload;
+			svc.fwd_relay_requested++;
+			return;
+		}
+	}
+	for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) {
+		if (svc_fwd_rel[i].tgt_ok) {
+			req->relay = true;
+			req->relay_index = (u8)i;
+			req->relay_kind = svc_fwd_rel[i].tgt_kind;
+			req->relay_page = svc_fwd_rel[i].tgt_page;
+			req->relay_off = svc_fwd_rel[i].tgt_off;
+			req->relay_payload = svc_fwd_rel[i].payload;
+			svc.fwd_relay_requested++;
+			return;
+		}
+	}
+}
+
+/*
  * Called only from the service thread (the pump). Returns true when a guest
  * (hardware-copy) request was queued; the caller then decides whether to hold
  * the entry for its deferred releases. All wave-1 gates are unchanged.
@@ -6209,6 +6503,12 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
 			    gsp_phys_guest_mem_enabled();
 	svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed &&
 			    svc_fwd_rel_n > 0;
+	/*
+	 * P6c: choose the one relayed record for this request (if any). The
+	 * selection is reset with the rest of the slot every time the releases
+	 * do not qualify, so a later pass can never replay a stale pick.
+	 */
+	svc_fwd_relay_select(&svc_fwd_req);
 	if (svc_fwd_req.guest) {
 		svc_fwd_req.vgpu = vgpu;
 		svc_fwd_req.src_kind = svc_fwd_cand.src_kind;
@@ -6360,6 +6660,42 @@ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft,
 		*canceled = svc.fwd_hold_canceled;
 }
 
+/*
+ * P6c completion-relay counters (see gsp_sema_service.h). Same plain-read
+ * discipline as the wave-1/1.5 stats above.
+ */
+void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback,
+				      u64 *skipped_sw, u64 *verify_fail,
+				      u64 *resolve_fail)
+{
+	if (requested)
+		*requested = svc.fwd_relay_requested;
+	if (ok)
+		*ok = svc.fwd_relay_ok;
+	if (fallback)
+		*fallback = svc.fwd_relay_fallback;
+	if (skipped_sw)
+		*skipped_sw = svc.fwd_relay_skipped_sw;
+	if (verify_fail)
+		*verify_fail = svc.fwd_relay_verify_fail;
+	if (resolve_fail)
+		*resolve_fail = svc.fwd_relay_resolve_fail;
+}
+
+/*
+ * P6c: true while this channel owns an active RELAY hold. The pump's
+ * per-pass notifier is suppressed for such a channel so the hold's verified
+ * success edge (or the next pass after a failure resolution) is the single
+ * completion post rather than a premature one before the guest release lands.
+ */
+static bool svc_fwd_relay_hold_active(struct nv_vgpu_device *vgpu,
+				      const struct svc_channel *ch)
+{
+	return svc_fwd_hold.active && svc_fwd_hold.relay_armed &&
+	       svc_fwd_hold.vgpu == vgpu &&
+	       svc_fwd_hold.ch_handle == ch->h_object;
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	/* Only ever called from the service thread. */
@@ -6414,7 +6750,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		u64 applied_before;
 		bool held = false;
 
-		svc_notify_complete(vgpu, ch);
+		/*
+		 * P6c: while a relay hold owns this channel, the hold's
+		 * verified success edge is the single completion post; skip
+		 * the per-pass notifier so it cannot fire before the guest
+		 * release is authored.
+		 */
+		if (!svc_fwd_relay_hold_active(vgpu, ch))
+			svc_notify_complete(vgpu, ch);
 
 		/*
 		 * P6b wave 1.5: enforce the hold cap on EVERY pass for the held
@@ -6729,6 +7072,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									svc_fwd_rel_apply_all(vgpu, ch,
 											      svc_fwd_rel,
 											      svc_fwd_rel_n);
+									/*
+									 * P6c: a
+									 * selected
+									 * relay whose
+									 * request did
+									 * not queue is
+									 * software-applied
+									 * here.
+									 */
+									if (svc_fwd_req.relay)
+										svc.fwd_relay_fallback++;
+									svc_fwd_req.relay = false;
 								}
 							}
 							/*
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 35d1c0f..ffb7df6 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -118,6 +118,20 @@ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft,
 				     u64 *canceled);
 
 /*
+ * P6c completion-relay counters for the gsp-phys-state work_relay line:
+ * requested relay selections armed on a deferred guest request, relays whose
+ * engine-authored guest semaphore was verified (ok) with their software
+ * applies skipped (skipped_sw), relays that fell back to the software apply
+ * (fallback: engine failure/timeout/abort or a word that did not carry the
+ * payload), hold re-reads that did not see the payload (verify_fail) and
+ * work-item semaphore HPA resolution failures (resolve_fail). All zero while
+ * gsp_phys_relay=0. Plain reads, safe from the debugfs show path.
+ */
+void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback,
+				      u64 *skipped_sw, u64 *verify_fail,
+				      u64 *resolve_fail);
+
+/*
  * Discover and pin GR (compute) channel rings after the CE region of the
  * channel video-memory allocation. Set false to leave GR rings untranslated
  * (the pump then only runs the CE channels).