vgsp: guest GR forwarding backend and capture (gsp_phys_gr_forward, default off)gr-forwarding

Add a default-off guest GR (compute) forwarding path so a guest CUDA
launch submitted on its GR channel can be executed by the physical GR
engine on the ladder channel, with fail-closed fallback to the host CUDA
bridge and the unchanged software release path.

- New module parameter gsp_phys_gr_forward (default 0) and accessor.
- New GR execution gate: boot_stage.rmcall_gr_exec_ok, latched only after
  a real GR submission logs REAL GR SEM OK; the backend refuses while it
  is unset.
- Sema capture: a completed 64-word inline QMD on a trusted, non-replay
  GR decode becomes an svc_gr_cand (push identity, decoded release0,
  parameter/program/pointer fields, up to 8 captured inline-write
  records); the CUDA bridge stays authoritative while the parameter is
  off and is reached in order forward -> bridge -> software release.
- Backend: gsp_phys_backend_guest_gr_submit() maps pages at their own
  guest VAs through fresh Tier-1 small-PT pages/PDE installs or relocates
  inline-captured pages to ladder shadow pages (Tier 2), builds
  SET_OBJECT + the inline-QMD method run + REPORT_SEMAPHORE on the
  compute subchannel, reuses the shared GPFIFO/USERD/doorbell publish
  tail (extracted from the CE submit path, CE bytes unchanged) and polls
  the QMD release and/or module semaphore, draining the epoch-gated
  fault queue on timeout.
- Sema wiring: GR requests reuse the INFLIGHT/hold/deferral machinery so
  the guest channel release applies only on the submit's success/failure
  edge; hold shims preserve the channel engine type.
- Counters and a work_gr state line plus a stats accessor.
- Bounds are fixed: 16 pages, 8 inline records, 64 QMD words; every
  pointer/size checked, pool exhaustion and stale mappings fail closed.
- New ABI layouts pinned with static_asserts.

Compile-only change: no device test performed.

6 files changed, 1622 insertions, 90 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index d5585e4..b26f2bd 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -609,6 +609,21 @@ bool gsp_phys_boot_requested(void)
 	((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
 
 /*
+ * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources,
+ * allocated lazily under gsp_phys_rm_lock and freed with the staging. The
+ * Tier-1 window pool holds small-PT pages installed with DMA_UPDATE_PDE_2 at
+ * the guest's own 2 MiB VA windows (one PDE per window); the mapper's
+ * conservative no-cached-line rule only allows a new PTE in a fetch line the
+ * current submission installed, and never rewrites a stale one. The Tier-2
+ * shadow chunk backs relocated parameter pages whose content was captured as
+ * inline writes. Both are deliberately outside rmcall_chunks so the COPY0
+ * stage-2 registration walk stays byte-identical.
+ */
+#define GSP_PHYS_RMCALL_GR_PT_PAGES	8u
+#define GSP_PHYS_RMCALL_GR_PT_ENTRIES	32u
+#define GSP_PHYS_RMCALL_GR_SHADOW_PAGES	4u
+
+/*
  * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
  * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
  * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
@@ -1418,6 +1433,29 @@ static struct gsp_phys_boot_stage {
 	 */
 	u64 rmcall_va_guest_pt;		/* guest scratch-PT VA window base */
 	u32 rmcall_guest_pt_next_line;	/* next never-used 128 B PTE line */
+	/*
+	 * GR forwarding (gsp_phys_gr_forward=1; inert otherwise). The exec
+	 * gate latches only after the stage-4 real GR submission logs
+	 * REAL GR SEM OK, so the forward path can never pretend to work while
+	 * the ladder itself cannot execute GR. The window pool and shadow
+	 * chunk are allocated lazily by the forwarding submit.
+	 */
+	bool rmcall_gr_exec_ok;		/* a real GR submission proved out */
+	u32 rmcall_gr_submit_epoch;	/* per-submission mapper epoch */
+	u32 rmcall_gr_pt_used;		/* Tier-1 window pool slots used */
+	struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES];
+	struct gsp_phys_boot_chunk rmcall_gr_shadow;	/* Tier-2 pages */
+	struct {
+		u32 pde_index;
+		u8 pt_slot;
+		u8 used;
+		u8 pte_count;
+		struct {
+			u32 idx;	/* small-PT index in the 2 MiB window */
+			u32 epoch;	/* submission that installed it */
+			u64 hpa;	/* the PTE's mapped host PA */
+		} pte[GSP_PHYS_RMCALL_GR_PT_ENTRIES];
+	} rmcall_gr_win[GSP_PHYS_RMCALL_GR_PT_PAGES];
 	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
 	struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
 	u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
@@ -2016,6 +2054,13 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_guest_selftest_ok = false;
 	boot_stage.rmcall_va_guest_pt = 0;
 	boot_stage.rmcall_guest_pt_next_line = 0;
+	boot_stage.rmcall_gr_exec_ok = false;
+	boot_stage.rmcall_gr_submit_epoch = 0;
+	boot_stage.rmcall_gr_pt_used = 0;
+	memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt));
+	memset(&boot_stage.rmcall_gr_shadow, 0,
+	       sizeof(boot_stage.rmcall_gr_shadow));
+	memset(boot_stage.rmcall_gr_win, 0, sizeof(boot_stage.rmcall_gr_win));
 	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
 		boot_stage.rmcall_mem_handles[i] = 0;
 		boot_stage.rmcall_mem_registered[i] = false;
@@ -9434,84 +9479,39 @@ struct gsp_phys_rmcall_ce_submit {
 };
 
 /*
- * One CE copy block on the ladder channel: clear the module semaphore, build
- * the methods for bytes into the module pushbuffer, publish the GPFIFO entry
- * and USERD GP_PUT and ring the doorbell. The caller must hold
- * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical
- * lock-across-poll; the P5 work API polls without the lock). Every guard of
- * the original stage-4 submission body is preserved here: server-memory
- * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the
- * token check. The semaphore is cleared before the doorbell so a stale release
- * from a previous run cannot be observed as this submission's completion.
- * P6c: sem_payload is the release value the method block carries; legacy
- * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is
- * still cleared even when the caller polls a guest page: the clear only
- * guards the module-sem poll path and is a harmless non-guest write.
- *
- * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release
- * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and
- * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call
- * site passes false, so the default COPY0 ladder emits the same bytes as
- * before.
+ * Shared publish tail of the ladder submissions: read USERD GP_PUT/GP_GET,
+ * encode and write the GPFIFO entry for the module pushbuffer VA, advance
+ * GP_PUT, snapshot the C3 fault epoch and ring the PF doorbell. Extracted
+ * verbatim from gsp_phys_rmcall_ce_submit_locked so the CE path and the GR
+ * launch path publish byte-identically; the CE caller's semaphore clear and
+ * method-block build happen before this call. The caller must hold
+ * gsp_phys_rm_lock.
  */
-static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
-					    u64 dst_va, u64 sem_va, u32 bytes,
-					    u32 sem_payload, bool gr,
-					    struct gsp_phys_rmcall_ce_submit *out)
+static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va,
+					  u32 words,
+					  struct gsp_phys_rmcall_ce_submit *out)
 {
-	struct gsp_phys_boot_chunk *pb =
-		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
 	struct gsp_phys_boot_chunk *ring =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO];
 	struct gsp_phys_boot_chunk *userd_c =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
+	struct gsp_phys_boot_chunk *pb =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
 	struct gsp_phys_boot_chunk *sem =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
 	struct gsp_phys_boot_chunk *src =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
 	struct gsp_phys_boot_chunk *dst =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
-	resource_size_t bar0_len;
-	u32 *pb_va, *userd, *ring_va;
-	u32 words, gpput, gpget, slot, e0, e1;
+	u32 *ring_va, *userd;
+	u32 gpput, gpget, slot, e0, e1;
 	u32 runlist_id, ch_id, doorbell;
-	u64 pb_gpu_va;
 
-	if (boot_stage.rmcall_channel_server_mem)
-		return -EOPNOTSUPP;
-	if (!boot_stage.pdev)
-		return -EINVAL;
-	bar0_len = pci_resource_len(boot_stage.pdev, 0);
-	if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4)
-		return -EIO;
-	if (!bar0 || !boot_stage.rmcall_token || !bytes ||
-	    bytes > GSP_PHYS_RMCALL_COPY_SIZE)
-		return -EINVAL;
-	pb_gpu_va = boot_stage.rmcall_va_pb;
-	if (!pb_gpu_va || (pb_gpu_va & 3))
-		return -EIO;
-
-	pb_va = gsp_phys_chunk_page_va(pb, 0);
 	ring_va = gsp_phys_chunk_page_va(ring, 0);
 	userd = gsp_phys_chunk_page_va(userd_c, 0);
-	if (!pb_va || !ring_va || !userd ||
-	    !gsp_phys_chunk_page_va(sem, 0) ||
-	    !gsp_phys_chunk_page_va(src, 0) ||
-	    !gsp_phys_chunk_page_va(dst, 0))
+	if (!ring_va || !userd)
 		return -EIO;
 
-	/* A stale release from a previous submission must not be observed. */
-	WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0);
-	wmb();
-
-	if (gr)
-		words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va,
-							 sem_payload);
-	else
-		words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va,
-						      dst_va, bytes,
-						      sem_payload);
-
 	gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
 	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
 	if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) {
@@ -9590,6 +9590,87 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
 }
 
 /*
+ * One CE copy block on the ladder channel: clear the module semaphore, build
+ * the methods for bytes into the module pushbuffer, publish the GPFIFO entry
+ * and USERD GP_PUT and ring the doorbell. The caller must hold
+ * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical
+ * lock-across-poll; the P5 work API polls without the lock). Every guard of
+ * the original stage-4 submission body is preserved here: server-memory
+ * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the
+ * token check. The semaphore is cleared before the doorbell so a stale release
+ * from a previous run cannot be observed as this submission's completion.
+ * P6c: sem_payload is the release value the method block carries; legacy
+ * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is
+ * still cleared even when the caller polls a guest page: the clear only
+ * guards the module-sem poll path and is a harmless non-guest write.
+ *
+ * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release
+ * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and
+ * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call
+ * site passes false, so the default COPY0 ladder emits the same bytes as
+ * before.
+ */
+static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
+					    u64 dst_va, u64 sem_va, u32 bytes,
+					    u32 sem_payload, bool gr,
+					    struct gsp_phys_rmcall_ce_submit *out)
+{
+	struct gsp_phys_boot_chunk *pb =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
+	struct gsp_phys_boot_chunk *ring =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO];
+	struct gsp_phys_boot_chunk *userd_c =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
+	struct gsp_phys_boot_chunk *sem =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+	struct gsp_phys_boot_chunk *src =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
+	struct gsp_phys_boot_chunk *dst =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
+	resource_size_t bar0_len;
+	u32 *pb_va, *userd, *ring_va;
+	u32 words;
+	u64 pb_gpu_va;
+
+	if (boot_stage.rmcall_channel_server_mem)
+		return -EOPNOTSUPP;
+	if (!boot_stage.pdev)
+		return -EINVAL;
+	bar0_len = pci_resource_len(boot_stage.pdev, 0);
+	if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4)
+		return -EIO;
+	if (!bar0 || !boot_stage.rmcall_token || !bytes ||
+	    bytes > GSP_PHYS_RMCALL_COPY_SIZE)
+		return -EINVAL;
+	pb_gpu_va = boot_stage.rmcall_va_pb;
+	if (!pb_gpu_va || (pb_gpu_va & 3))
+		return -EIO;
+
+	pb_va = gsp_phys_chunk_page_va(pb, 0);
+	ring_va = gsp_phys_chunk_page_va(ring, 0);
+	userd = gsp_phys_chunk_page_va(userd_c, 0);
+	if (!pb_va || !ring_va || !userd ||
+	    !gsp_phys_chunk_page_va(sem, 0) ||
+	    !gsp_phys_chunk_page_va(src, 0) ||
+	    !gsp_phys_chunk_page_va(dst, 0))
+		return -EIO;
+
+	/* A stale release from a previous submission must not be observed. */
+	WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0);
+	wmb();
+
+	if (gr)
+		words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va,
+							 sem_payload);
+	else
+		words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va,
+						      dst_va, bytes,
+						      sem_payload);
+
+	return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out);
+}
+
+/*
  * H1 stage-4 GR verification: one GR semaphore release on the ladder channel,
  * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller
  * must hold gsp_phys_rm_lock (stage 4 keeps its historical lock-across-poll;
@@ -9660,6 +9741,12 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
 		poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget,
 		get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
 		sub.words, boot_stage.rmcall_token);
+	/*
+	 * Latch the GR execution gate: this is the only place a real GR
+	 * submission is proven on the ladder, and gsp_phys_backend_gr_ready()
+	 * refuses every forwarded launch until this is observed.
+	 */
+	boot_stage.rmcall_gr_exec_ok = true;
 	return 0;
 }
 
@@ -9769,6 +9856,36 @@ bool gsp_phys_backend_work_ready(void)
 }
 EXPORT_SYMBOL_GPL(gsp_phys_backend_work_ready);
 
+/*
+ * GR forwarding readiness: the parameter, a live client-memory GR ladder
+ * channel and the latched boot proof that a real GR submission executed
+ * (rmcall_gr_exec_ok, set only by a REAL GR SEM OK). This is deliberately
+ * separate from gsp_phys_backend_ladder_ready(), which refuses CE work on a
+ * GR0 ladder; a GR0 ladder is exactly the GR forwarding prerequisite. Every
+ * submit re-checks the same state under gsp_phys_rm_lock.
+ */
+bool gsp_phys_backend_gr_ready(void)
+{
+	if (!gsp_phys_gr_forward_enabled())
+		return false;
+	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY)
+		return false;
+	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
+	    boot_stage.rm_unload_submitted)
+		return false;
+	if (!gsp_phys_rmcall_engine_is_gr() || !boot_stage.rmcall_gr_exec_ok)
+		return false;
+	if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem ||
+	    !boot_stage.rmcall_token)
+		return false;
+	if (!boot_stage.rmcall_va_pb || !boot_stage.rmcall_va_gpfifo ||
+	    !boot_stage.rmcall_va_src || !boot_stage.rmcall_va_sem ||
+	    !boot_stage.rmcall_pdb_root)
+		return false;
+	return true;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_ready);
+
 bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va,
 				    u32 *bytes)
 {
@@ -10525,6 +10642,595 @@ int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
 }
 EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
 
+/* ------------------------------------------------------------------ */
+/* GR (compute) forwarding backend (gsp_phys_gr_forward)               */
+/* ------------------------------------------------------------------ */
+
+/*
+ * C5C0 inline-QMD method window: SET_INLINE_QMD_ADDRESS_A/B carry the
+ * shifted QMD device address and LOAD_INLINE_QMD_DATA the 64 QMD words; the
+ * guest emits the window as one INC run at 0x318 with count 66 (A, B, then 64
+ * words), and the launch is replayed in exactly that shape with the QMD words
+ * the sema service decoded (release0 and, under Tier 2, relocated pointers
+ * rewritten). The channel release is the same A..D block as the H1 GR sem
+ * verify, with a payload distinct from the CE/module sentinel.
+ */
+#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A	0x318u
+#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_B	0x31cu
+#define GSP_PHYS_RMCALL_GR_LOAD_INLINE_QMD_DATA	0x320u
+/* Header count: ADDRESS_A + ADDRESS_B + 64 QMD words. */
+#define GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS	66u
+#define GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD	GSP_PHYS_RMCALL_SEM_PAYLOAD
+#define GSP_PHYS_RMCALL_GR_SEM_PAYLOAD		0xc0ffee02u
+/* Method block bound: SET_OBJECT(2) + inline run(67) + REPORT_SEM(8). */
+#define GSP_PHYS_RMCALL_GR_METHOD_WORDS		96u
+
+static u64 gsp_phys_gr_submitted;
+static u64 gsp_phys_gr_ok;
+static u64 gsp_phys_gr_refused;
+static u64 gsp_phys_gr_timed_out;
+static u64 gsp_phys_gr_pool_exhausted;
+
+void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused,
+				  u64 *timed_out, u64 *pool_exhausted)
+{
+	if (submitted)
+		*submitted = gsp_phys_gr_submitted;
+	if (ok)
+		*ok = gsp_phys_gr_ok;
+	if (refused)
+		*refused = gsp_phys_gr_refused;
+	if (timed_out)
+		*timed_out = gsp_phys_gr_timed_out;
+	if (pool_exhausted)
+		*pool_exhausted = gsp_phys_gr_pool_exhausted;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_counters);
+
+/*
+ * GR launch method block: SET_OBJECT on the compute subchannel, the inline
+ * QMD run at its original shape and the channel REPORT_SEMAPHORE release.
+ * Returns the block size in dwords.
+ */
+static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const u32 *qmd,
+					   u64 qmd_va, u64 sem_va,
+					   u32 sem_payload)
+{
+	u32 *p = pb;
+
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT,
+				    GSP_PHYS_RMCALL_CLASS_COMPUTE);
+	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+	       (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) |
+	       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
+	       (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2);
+	*p++ = (u32)(qmd_va >> 40);
+	*p++ = (u32)((qmd_va >> 8) & 0xffffffffu);
+	memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
+	p += GSP_PHYS_GR_QMD_WORDS;
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A,
+				    (u32)(sem_va >> 32));
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B,
+				    (u32)sem_va);
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C,
+				    sem_payload);
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D,
+				    GSP_PHYS_RMCALL_GR_SEM_RELEASE_D);
+	return (u32)(p - pb);
+}
+
+/*
+ * Tier-1 window pool: find (or install) the small-PT page for one guest
+ * 2 MiB VA window through DMA_UPDATE_PDE_2. Installation is once per window
+ * per staging, so a later launch in the same window writes PTEs into the same
+ * server-installed PT page. Called with gsp_phys_rm_lock held.
+ */
+static int gsp_phys_rmcall_gr_win_slot_locked(void __iomem *bar0,
+					      u32 pde_index, u32 *slot_out)
+{
+	struct gsp_phys_rmcall_update_pde2_params p;
+	struct gsp_phys_boot_chunk *pt;
+	u32 rpc_result = 0, status = 0;
+	u32 i, slot;
+	int ret;
+
+	for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) {
+		if (boot_stage.rmcall_gr_win[i].used &&
+		    boot_stage.rmcall_gr_win[i].pde_index == pde_index) {
+			*slot_out = i;
+			return 0;
+		}
+	}
+	if (boot_stage.rmcall_gr_pt_used >= GSP_PHYS_RMCALL_GR_PT_PAGES)
+		return -ENOSPC;
+	slot = boot_stage.rmcall_gr_pt_used;
+	pt = &boot_stage.rmcall_gr_pt[slot];
+	if (!pt->pages) {
+		ret = gsp_phys_chunk_alloc(boot_stage.pdev, pt, 1u);
+		if (ret)
+			return ret;
+		gsp_phys_chunk_zero(pt);
+	}
+	memset(&p, 0, sizeof(p));
+	p.pde_index = cpu_to_le32(pde_index);
+	p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
+	p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0));
+	p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
+	p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
+	p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
+	p.pde_buffer = 0;
+	p.sub_device_id = 0;
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
+			GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
+			&p, sizeof(p), sizeof(p), &p, sizeof(p),
+			&rpc_result, &status, "GR forward PDE_UPDATE");
+	if (ret) {
+		pr_err("vgsp-gr: forward PDE install refused pdeIndex=%u (window VA 0x%llx) status=0x%x rpc_result=0x%x ret=%d\n",
+		       pde_index, (u64)pde_index << 21, status, rpc_result, ret);
+		return ret;
+	}
+	boot_stage.rmcall_gr_win[slot].pde_index = pde_index;
+	boot_stage.rmcall_gr_win[slot].pt_slot = (u8)slot;
+	boot_stage.rmcall_gr_win[slot].used = true;
+	boot_stage.rmcall_gr_win[slot].pte_count = 0;
+	boot_stage.rmcall_gr_pt_used = slot + 1;
+	pr_info("vgsp-gr: forward PDE installed pdeIndex=%u slot=%u ptPa=0x%llx status=0x%x\n",
+		pde_index, slot, (u64)gsp_phys_chunk_page_dma(pt, 0), status);
+	*slot_out = slot;
+	return 0;
+}
+
+/*
+ * Tier-1 map: write one guest page's HPA into the PT page of its own 2 MiB
+ * VA window at the guest VA itself, so the QMD and every pointer inside the
+ * mapped pages resolve without rewriting. The GMMU caches 128 B PTE fetch
+ * lines with no invalidate, so the mapper never rewrites an installed PTE
+ * (a different HPA for the same VA is refused rather than made stale) and
+ * only adds a new PTE to a fetch line this same submission already touched.
+ * Every refusal is fail-closed: the caller drops to Tier 2 or refuses the
+ * launch and the sema service falls back. Called with gsp_phys_rm_lock held.
+ */
+static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va,
+					    u64 hpa, u32 epoch)
+{
+	u64 page = guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+	u32 pde_index = (u32)(page >> 21);
+	u32 pte_idx = (u32)((page >> 12) & 0x1ffu);
+	u32 line = pte_idx / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES;
+	struct gsp_phys_boot_chunk *pt;
+	u8 *pt_va;
+	u32 slot, i;
+	int ret;
+
+	ret = gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot);
+	if (ret)
+		return ret;
+	for (i = 0; i < boot_stage.rmcall_gr_win[slot].pte_count; i++) {
+		if (boot_stage.rmcall_gr_win[slot].pte[i].idx == pte_idx) {
+			if (boot_stage.rmcall_gr_win[slot].pte[i].hpa == hpa)
+				return 0;	/* already mapped, no write */
+			return -EALREADY;	/* stale: never rewrite */
+		}
+		if (boot_stage.rmcall_gr_win[slot].pte[i].epoch != epoch &&
+		    boot_stage.rmcall_gr_win[slot].pte[i].idx /
+			    GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES == line)
+			return -EBUSY;	/* fetch line may be cached */
+	}
+	if (boot_stage.rmcall_gr_win[slot].pte_count >=
+	    GSP_PHYS_RMCALL_GR_PT_ENTRIES)
+		return -ENOSPC;
+	pt = &boot_stage.rmcall_gr_pt[boot_stage.rmcall_gr_win[slot].pt_slot];
+	pt_va = gsp_phys_chunk_page_va(pt, 0);
+	if (!pt_va)
+		return -EIO;
+	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
+			   pt_va + (size_t)pte_idx * 8);
+	dma_wmb();
+	boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx;
+	boot_stage.rmcall_gr_win[slot].pte[i].epoch = epoch;
+	boot_stage.rmcall_gr_win[slot].pte[i].hpa = hpa;
+	boot_stage.rmcall_gr_win[slot].pte_count++;
+	return 0;
+}
+
+/*
+ * Tier-2 map: consume one never-reused 128 B fetch line from the guest
+ * scratch PT pool (PTE0 only) for a ladder-owned shadow page and return the
+ * line's base VA. The shadow chunk is allocated lazily; page slot `slot`
+ * supplies the backing page. Called with gsp_phys_rm_lock held.
+ */
+static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa,
+						u64 *va_out)
+{
+	struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow;
+	struct gsp_phys_boot_chunk *pt;
+	u32 line = boot_stage.rmcall_guest_pt_next_line;
+	u32 page, line_in_page;
+	u64 va;
+	u8 *pt_va;
+	int ret;
+
+	if (!sh->pages) {
+		ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
+					   GSP_PHYS_RMCALL_GR_SHADOW_PAGES);
+		if (ret)
+			return ret;
+		gsp_phys_chunk_zero(sh);
+	}
+	if (slot >= sh->pages)
+		return -ENOSPC;
+	if (!boot_stage.rmcall_va_guest_pt)
+		return -EIO;
+	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
+		return -ENOSPC;
+	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
+	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
+	pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
+	if (!pt->pages)
+		return -EIO;
+	pt_va = gsp_phys_chunk_page_va(pt, 0);
+	if (!pt_va)
+		return -EIO;
+	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
+			   pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+					    line_in_page) * 8);
+	dma_wmb();
+	boot_stage.rmcall_guest_pt_next_line = line + 1;
+	va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
+	     ((u64)line_in_page << 16);
+	*va_out = va;
+	return 0;
+}
+
+/* GR forward fault observation is armed by the GR parameters, not the CE
+ * ones (gsp_phys_fault_relay_active() requires gsp_phys_work_forward). */
+static bool gsp_phys_gr_fault_active(void)
+{
+	return gsp_phys_fault_relay_enabled() &&
+	       gsp_phys_gr_forward_enabled() &&
+	       gsp_phys_guest_mem_enabled();
+}
+
+/*
+ * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The
+ * caller has resolved the launch's pages; every mapping is fresh or proven
+ * unchanged, the QMD release0 and the channel release are authored either by
+ * the engine into ladder-mapped shadow pages or into the module semaphore,
+ * and the GPFIFO/USERD/doorbell tail is the shared publish helper. The
+ * release poll runs without gsp_phys_rm_lock; a timeout drains the status
+ * queue (epoch-gated) into out->fault. Every refusal moves the GR counters
+ * and returns an errno the sema service falls back on.
+ */
+int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
+				     struct gsp_phys_backend_copy_result *out)
+{
+	struct gsp_phys_boot_chunk *sem =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+	struct gsp_phys_boot_chunk *pb =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
+	void __iomem *bar0;
+	struct gsp_phys_rmcall_ce_submit sub;
+	struct gsp_phys_fault_info fault = { };
+	/*
+	 * Static scratch: the only caller is the single-inflight sema work
+	 * item (ordered gsp-phys-work queue), so no concurrent submission can
+	 * touch these. Keeping the 96-dword method block and the 64-word QMD
+	 * copy off the stack keeps the frame under the warning bound.
+	 */
+	static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS];
+	static u32 qmd[GSP_PHYS_GR_QMD_WORDS];
+	unsigned long start = jiffies;
+	unsigned long deadline;
+	u8 *sem_va;
+	u32 epoch, words, i, n_shadow = 0;
+	u32 want_qmd = 0, want_ch = 0;
+	bool qmd_rel = false, ch_rel = false;
+	u64 qmd_rel_hpa = 0, ch_rel_hpa = 0;
+	u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4;
+	u32 qmd_rel_off = 0, ch_rel_off = 0;
+	u32 poll = 0, w0 = 0, w1 = 0;
+	int ret = 0;
+
+	if (out)
+		memset(out, 0, sizeof(*out));
+	if (!req || !req->qmd ||
+	    req->qmd_words != GSP_PHYS_GR_QMD_WORDS ||
+	    req->n_pages > GSP_PHYS_GR_MAX_PAGES ||
+	    req->n_inl > GSP_PHYS_GR_MAX_INLINE) {
+		gsp_phys_gr_refused++;
+		return -EINVAL;
+	}
+	if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() ||
+	    gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) {
+		gsp_phys_gr_refused++;
+		ret = -EAGAIN;
+		goto out;
+	}
+	bar0 = gsp_phys_gpu_bar0();
+	if (!bar0) {
+		gsp_phys_gr_refused++;
+		ret = -EAGAIN;
+		goto out;
+	}
+	sem_va = gsp_phys_chunk_page_va(sem, 0);
+	if (!sem_va || !gsp_phys_chunk_page_va(pb, 0)) {
+		gsp_phys_gr_refused++;
+		ret = -EIO;
+		goto out;
+	}
+
+	gsp_phys_gr_submitted++;
+	mutex_lock(&gsp_phys_rm_lock);
+	if (!gsp_phys_backend_gr_ready()) {
+		mutex_unlock(&gsp_phys_rm_lock);
+		gsp_phys_gr_refused++;
+		ret = -EAGAIN;
+		goto out;
+	}
+	epoch = ++boot_stage.rmcall_gr_submit_epoch;
+	if (!epoch)
+		epoch = ++boot_stage.rmcall_gr_submit_epoch;
+	memcpy(qmd, req->qmd, sizeof(qmd));
+
+	/*
+	 * QMD release0: when the launch has a one-word absolute release and
+	 * the caller resolved the guest release page, author it there through
+	 * a fresh shadow line so the guest observes its own QMD release; else
+	 * point release0 at the module semaphore so the host observes engine
+	 * completion. A reduction release is never redirected (its arithmetic
+	 * would change).
+	 */
+	if (((qmd[4] >> 10) & 1)) {
+		u32 red_en = (qmd[24] >> 26) & 1;
+		u32 one_word = (qmd[24] >> 31) & 1;
+
+		if (req->rel_hpa && one_word && !red_en &&
+		    n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
+			u64 sh_va = 0;
+
+			ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow,
+								   req->rel_hpa,
+								   &sh_va);
+			if (!ret) {
+				n_shadow++;
+				qmd_rel_hpa = req->rel_hpa;
+				qmd_rel_off = req->rel_off;
+				want_qmd = req->rel_payload;
+				qmd_rel = true;
+				qmd[23] = (u32)(sh_va + req->rel_off);
+				qmd[24] = (qmd[24] & 0xffffff00u) |
+					  ((u32)((sh_va + req->rel_off) >> 32) &
+					   0xffu);
+				qmd[25] = req->rel_payload;
+			} else {
+				ret = 0;
+			}
+		}
+		if (!qmd_rel) {
+			qmd[23] = (u32)GSP_PHYS_RMCALL_VA_SEM;
+			qmd[24] = (qmd[24] & 0xffffff00u) |
+				  ((u32)(GSP_PHYS_RMCALL_VA_SEM >> 32) &
+				   0xffu);
+			qmd[25] = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD;
+			want_qmd = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD;
+			qmd_rel = true;
+		}
+	}
+
+	/*
+	 * Pages: Tier 1 maps the resolved HPA at the guest VA; a page without
+	 * a usable Tier-1 mapping falls to Tier 2 when captured inline bytes
+	 * cover it and a QMD pointer pair names it (the pair is rewritten to
+	 * the shadow VA); anything else refuses the launch.
+	 */
+	for (i = 0; i < req->n_pages; i++) {
+		const struct gsp_phys_gr_page *pg = &req->pages[i];
+		u64 pg_va, sh_va = 0;
+		bool mapped = false;
+
+		if (!pg->guest_va) {
+			mutex_unlock(&gsp_phys_rm_lock);
+			gsp_phys_gr_refused++;
+			ret = -EINVAL;
+			goto out;
+		}
+		pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+		if (pg->hpa) {
+			ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va,
+							       pg->hpa, epoch);
+			if (!ret)
+				mapped = true;
+			else if (ret == -ENOSPC)
+				gsp_phys_gr_pool_exhausted++;
+		}
+		if (!mapped && pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE &&
+		    pg->qmd_word + 1 < GSP_PHYS_GR_QMD_WORDS) {
+			struct gsp_phys_boot_chunk *sh =
+				&boot_stage.rmcall_gr_shadow;
+			u8 *sh_page;
+			u32 j, copied = 0;
+
+			if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
+				gsp_phys_gr_pool_exhausted++;
+				continue;
+			}
+			if (!sh->pages) {
+				ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
+							   GSP_PHYS_RMCALL_GR_SHADOW_PAGES);
+				if (ret) {
+					gsp_phys_gr_pool_exhausted++;
+					continue;
+				}
+				gsp_phys_chunk_zero(sh);
+			}
+			sh_page = gsp_phys_chunk_page_va(sh, n_shadow);
+			if (!sh_page)
+				continue;
+			memset(sh_page, 0, GSP_PHYS_PAGE_SIZE);
+			for (j = 0; j < req->n_inl; j++) {
+				const struct gsp_phys_gr_inline *inl =
+					&req->inl[j];
+				u32 len;
+
+				if ((inl->dst_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))
+				    != pg_va)
+					continue;
+				len = inl->len;
+				if (len > GSP_PHYS_GR_INLINE_BYTES)
+					len = GSP_PHYS_GR_INLINE_BYTES;
+				if ((u64)(inl->dst_va &
+					  (GSP_PHYS_PAGE_SIZE - 1)) + len >
+				    GSP_PHYS_PAGE_SIZE)
+					continue;
+				memcpy(sh_page + (inl->dst_va &
+						  (GSP_PHYS_PAGE_SIZE - 1)),
+				       inl->data, len);
+				copied++;
+			}
+			if (copied) {
+				ret = gsp_phys_rmcall_gr_map_shadow_locked(
+					n_shadow,
+					(u64)gsp_phys_chunk_page_dma(sh,
+								      n_shadow),
+					&sh_va);
+				if (!ret) {
+					u64 ptr_va = sh_va +
+						(pg->guest_va &
+						 (GSP_PHYS_PAGE_SIZE - 1));
+
+					n_shadow++;
+					qmd[pg->qmd_word] = (u32)ptr_va;
+					qmd[pg->qmd_word + 1] =
+						(qmd[pg->qmd_word + 1] &
+						 ~0x1ffffu) |
+						((u32)(ptr_va >> 32) & 0x1ffffu);
+					mapped = true;
+				}
+			}
+		}
+		if (!mapped) {
+			mutex_unlock(&gsp_phys_rm_lock);
+			gsp_phys_gr_refused++;
+			ret = -EOPNOTSUPP;
+			pr_info("vgsp-gr: launch refused: page va=0x%llx hpa=0x%llx qmd_word=%u has no Tier-1 mapping and no captured inline data (pages=%u)\n",
+				pg->guest_va, pg->hpa, pg->qmd_word,
+				req->n_pages);
+			goto out;
+		}
+	}
+
+	/*
+	 * Channel release: author it into the guest semaphore page when the
+	 * caller resolved one, else into the module semaphore's second word.
+	 */
+	if (req->sem_hpa && req->sem_payload &&
+	    n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
+		u64 sh_va = 0;
+
+		ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow,
+							   req->sem_hpa, &sh_va);
+		if (!ret) {
+			n_shadow++;
+			ch_rel_hpa = req->sem_hpa;
+			ch_rel_off = req->sem_off;
+			want_ch = req->sem_payload;
+			ch_rel = true;
+			ch_sem_va = sh_va + req->sem_off;
+		}
+	}
+	if (!ch_rel) {
+		want_ch = GSP_PHYS_RMCALL_GR_SEM_PAYLOAD;
+		ch_rel = true;
+	}
+
+	/* A stale release from a previous submission must not be observed. */
+	WRITE_ONCE(*(u32 *)sem_va, 0);
+	WRITE_ONCE(*(u32 *)(sem_va + 4), 0);
+	wmb();
+
+	words = gsp_phys_rmcall_build_gr_launch(methods, qmd,
+						GSP_PHYS_RMCALL_VA_SRC,
+						ch_sem_va, want_ch);
+	memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
+
+	ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb,
+					     words, &sub);
+	mutex_unlock(&gsp_phys_rm_lock);
+	if (ret) {
+		gsp_phys_gr_refused++;
+		pr_err("vgsp-gr: launch submit refused (%d) pages=%u words=%u\n",
+		       ret, req->n_pages, words);
+		goto out;
+	}
+
+	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
+		req->h_object, req->push_va, req->push_len, req->n_pages,
+		n_shadow, req->qmd_va, sub.slot, words, boot_stage.rmcall_token,
+		qmd_rel, want_ch, GSP_PHYS_RMCALL_GUEST_POLL_MS);
+
+	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS);
+	for (;;) {
+		if (qmd_rel) {
+			if (qmd_rel_hpa)
+				w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) +
+							qmd_rel_off));
+			else
+				w0 = READ_ONCE(*(u32 *)sem_va);
+		}
+		if (ch_rel_hpa)
+			w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) +
+						ch_rel_off));
+		else
+			w1 = READ_ONCE(*(u32 *)(sem_va + 4));
+		if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch)
+			break;
+		if (time_after_eq(jiffies, deadline)) {
+			gsp_phys_gr_timed_out++;
+			pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n",
+			       GSP_PHYS_RMCALL_GUEST_POLL_MS, w0, want_qmd, w1,
+			       want_ch, req->h_object, req->push_va);
+			if (gsp_phys_gr_fault_active()) {
+				struct gsp_phys_fault_info fi;
+				u32 fep = 0;
+
+				if (!gsp_phys_status_drain_fault(&fi, &fep) &&
+				    fi.valid &&
+				    fi.function ==
+				    GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) {
+					if (fep > READ_ONCE(gsp_phys_fault_submit_epoch)) {
+						fault = fi;
+						gsp_phys_fault_drained++;
+					} else {
+						gsp_phys_fault_stale++;
+					}
+				}
+			}
+			poll = w1;
+			ret = -ETIMEDOUT;
+			goto out;
+		}
+		msleep(GSP_PHYS_RMCALL_POLL_MS);
+	}
+
+	gsp_phys_gr_ok++;
+	poll = w1;
+	pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n",
+		req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1,
+		jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok);
+	ret = 0;
+
+out:
+	if (out) {
+		out->sem_value = poll;
+		out->elapsed_ms = jiffies_to_msecs(jiffies - start);
+		out->rc = ret;
+		out->fault = fault;
+	}
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_gr_submit);
+
 /*
  * P6a trigger variant: identical to the exported forwarding call except that
  * the gsp_phys_work_forward parameter gate is dropped (every other guard and
@@ -11296,6 +12002,9 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx);
 	for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++)
 		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx_pt[i]);
+	for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++)
+		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]);
+	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs);
@@ -11529,6 +12238,30 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 	}
 	{
 		/*
+		 * GR (compute) forwarding: the sema service's capture/offer/
+		 * fallback counters and the booter backend's submit/refusal
+		 * counters plus the parameter and exec-gate state. All zero
+		 * while gsp_phys_gr_forward=0.
+		 */
+		u64 gr_req = 0, gr_ok = 0, gr_fallback = 0, gr_refused = 0;
+		u64 gr_timeout = 0, gr_fault = 0;
+		u64 b_sub = 0, b_ok = 0, b_refused = 0, b_timeout = 0;
+		u64 b_pool = 0;
+
+		gsp_sema_service_fwd_gr_stats(&gr_req, &gr_ok, &gr_fallback,
+					      &gr_refused, &gr_timeout,
+					      &gr_fault);
+		gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused,
+					     &b_timeout, &b_pool);
+		seq_printf(m,
+			   "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u\n",
+			   gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout,
+			   gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool,
+			   gsp_phys_gr_forward_enabled(),
+			   boot_stage.rmcall_gr_exec_ok);
+	}
+	{
+		/*
 		 * P6c C3 fault observation/relay: the booter's 0x1006
 		 * observation counters (seen/stale/drained/wait_unmatched,
 		 * epoch) and the sema service's delivery counters
@@ -12835,6 +13568,17 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX +
 		      (u64)GSP_PHYS_RMCALL_GR_CTX_PT_MAX * (2ULL << 20) <=
 		      GSP_PHYS_RMCALL_VAS_SIZE,
 	      "the GR ctx window must stay below the VAS vaSize limit");
+static_assert(sizeof(struct gsp_phys_gr_page) == 24,
+	      "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes");
+static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 &&
+	      sizeof(struct gsp_phys_gr_inline) == 80,
+	      "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes");
+static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <=
+	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
+	      "the GR launch method block (SET_OBJECT + inline-QMD run + REPORT_SEMAPHORE) must fit the bound");
+static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <=
+	      GSP_PHYS_RMCALL_COPY_SIZE,
+	      "the GR launch method block must fit one module pushbuffer page");
 
 /*
  * GspSystemInfo wire ABI, pinned against the reference header compile
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index f462c4e..73c21ad 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -296,6 +296,92 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
 				     u64 *refused, u64 *pool_exhausted);
 
 /*
+ * GR (compute) guest forwarding (gsp_phys_gr_forward=1 AND
+ * gsp_phys_guest_mem=1; both default off). The caller has already decoded a
+ * guest GR launch, resolved the guest VAs of the QMD's program/parameter/
+ * pointer pages to guest PAs, and resolved those PAs to host PAs.
+ *
+ * gsp_phys_gr_page describes one page the launch references: its guest VA
+ * (where Tier 1 maps the HPA), the host PA (0 = unresolved) and the QMD word
+ * index of the 64-bit pointer pair that names it (qmd_word < 48, 32 for the
+ * parameter bank, 48 for the program, GSP_PHYS_GR_QMD_WORD_NONE when no QMD
+ * word points at the page). A page with an unresolved HPA can still be
+ * submitted through Tier 2 when the caller supplies captured inline-write
+ * bytes for it and qmd_word points at it: the backend allocates a ladder
+ * shadow page, copies the captured bytes and rewrites that QMD word pair.
+ *
+ * gsp_phys_gr_inline is one captured GRCE inline write: the destination guest
+ * VA and its captured payload bytes.
+ *
+ * gsp_phys_gr_submit is one launch: the 64 rewritten-QMD source words, the
+ * optional guest-visible release page for QMD release0 (rel_hpa/rel_off/
+ * rel_payload; zero rel_hpa points release0 at the module semaphore), the
+ * optional guest-visible channel release page (sem_hpa/sem_off/sem_payload;
+ * zero sem_hpa points the REPORT_SEMAPHORE block at the module semaphore) and
+ * push identity for logging.
+ *
+ * gsp_phys_backend_gr_submit() consumes only the fresh, never-reused 128 B
+ * PTE fetch lines (one per shadowed page and one per guest release page; the
+ * GMMU caches PTE fetches at 128 B with no invalidate), refuses unless the
+ * GR exec gate is latched and the guest-RAM/identity-domain gates are open,
+ * builds SET_OBJECT + the inline-QMD loader + REPORT_SEMAPHORE on the compute
+ * subchannel, publishes the GPFIFO entry/USERD/doorbell and polls the module
+ * semaphore (and the QMD release word when rewritten to a ladder VA). On
+ * timeout it drains the epoch-gated fault queue into out->fault, exactly like
+ * the CE relay path. Every failure returns an errno the sema service falls
+ * back to; no guest state is lost. Returns 0 on a completed launch.
+ */
+#define GSP_PHYS_GR_MAX_PAGES		16u
+#define GSP_PHYS_GR_MAX_INLINE		8u
+#define GSP_PHYS_GR_INLINE_BYTES	64u
+#define GSP_PHYS_GR_QMD_WORDS		64u
+#define GSP_PHYS_GR_QMD_WORD_NONE	0xffffffffu
+
+struct gsp_phys_gr_page {
+	u64 guest_va;		/* guest VA (page aligned) */
+	u64 hpa;		/* host PA (page aligned; 0 = unresolved) */
+	u32 qmd_word;		/* QMD word of the pointer pair, or NONE */
+	u32 reserved;
+};
+
+struct gsp_phys_gr_inline {
+	u64 dst_va;		/* captured GRCE inline-write destination */
+	u32 len;		/* payload bytes (<= GSP_PHYS_GR_INLINE_BYTES) */
+	u8 data[GSP_PHYS_GR_INLINE_BYTES];
+};
+
+struct gsp_phys_gr_submit {
+	const u32 *qmd;		/* 64 rewritten-QMD source words */
+	u32 qmd_words;		/* must be GSP_PHYS_GR_QMD_WORDS */
+	u64 qmd_va;		/* guest VA of the QMD (logging only) */
+	const struct gsp_phys_gr_page *pages;
+	u32 n_pages;
+	const struct gsp_phys_gr_inline *inl;
+	u32 n_inl;
+	u64 rel_hpa;		/* guest-visible QMD release0 page, or 0 */
+	u32 rel_off;
+	u32 rel_payload;
+	u64 sem_hpa;		/* guest-visible channel release page, or 0 */
+	u32 sem_off;
+	u32 sem_payload;
+	u64 push_va;		/* guest push identity (logging only) */
+	u32 push_len;
+	u32 h_object;		/* guest GR channel handle (logging only) */
+};
+
+/* True when gsp_phys_gr_forward is set and the ladder is a live GR channel
+ * whose boot submission proved out (rmcall_gr_exec_ok). */
+bool gsp_phys_backend_gr_ready(void);
+
+int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
+				     struct gsp_phys_backend_copy_result *out);
+
+/* GR forwarding counters (submitted/verified/cancelled), plain reads. */
+void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused,
+				  u64 *timed_out, u64 *pool_exhausted);
+
+
+/*
  * P6c C3 fault observation/relay (gsp_phys_fault_relay=1; default off).
  *
  * gsp_phys_status_drain_fault() walks the unconsumed status-queue records with
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 74e3103..b204961 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -92,6 +92,18 @@ bool gsp_phys_srv_ctxbuf_value(void);
 bool gsp_phys_work_forward_enabled(void);
 
 /*
+ * True when the gsp_phys_gr_forward module parameter is set (default 0: guest
+ * GR (compute) launches stay on the existing host CUDA bridge / software
+ * path). When set, the sema service captures a completed guest inline QMD and
+ * the booter submits it as real work on the ladder's physical GR channel,
+ * but only after boot_stage.rmcall_gr_exec_ok proved a real GR submission at
+ * boot (gsp_phys_backend_gr_ready() enforces the gate). Every forward
+ * refusal/failure falls back to the CUDA bridge (when enabled) and then the
+ * unchanged software release path. One-line rollback: gsp_phys_gr_forward=0.
+ */
+bool gsp_phys_gr_forward_enabled(void);
+
+/*
  * True when the gsp_phys_guest_mem module parameter is set (default 0: the
  * debugfs rm-call ladder stays entirely on module-owned buffers, exactly as
  * before). When set, ladder triggers 6 (W2G) and 7 (G2H) may point the real
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index f897ce0..2b7fa80 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -611,6 +611,23 @@ struct sema_svc {
 	u64			fwd_fault_delivered;
 	u64			fwd_fault_no_event;
 	u64			fwd_fault_late_release;
+	/*
+	 * GR (compute) forwarding (gsp_phys_gr_forward=1 only): candidate
+	 * launches offered to the request path (fwd_gr_req), launches the
+	 * physical GR channel completed (fwd_gr_ok), offerings that fell back
+	 * to the host CUDA bridge / software release (fwd_gr_fallback), gate
+	 * refusals (fwd_gr_refused), backend timeouts (fwd_gr_timeout),
+	 * timeouts carrying a decoded engine fault (fwd_gr_fault) and CUDA
+	 * bridge launch calls made by a fallback (fwd_gr_bridge). All zero
+	 * while the parameter is off.
+	 */
+	u64			fwd_gr_req;
+	u64			fwd_gr_ok;
+	u64			fwd_gr_fallback;
+	u64			fwd_gr_refused;
+	u64			fwd_gr_timeout;
+	u64			fwd_gr_fault;
+	u64			fwd_gr_bridge;
 
 	/* Distinct stalled VA tracking for rate-limited logging. */
 #define SVC_STALLED_MAX		256
@@ -3154,6 +3171,215 @@ struct svc_fwd_cand {
 static struct svc_fwd_cand svc_fwd_cand;
 
 /*
+ * GR (compute) forwarding candidate (gsp_phys_gr_forward=1 only). When a GR
+ * channel decodes a completed 64-word inline QMD, the launch is captured here
+ * instead of being handed straight to the host CUDA bridge: the QMD words, its
+ * device VA, the push identity, the newest GRCE inline-write records and the
+ * QMD fields the bridge decodes (release0, parameter bank, program, pointer
+ * pairs). svc_fwd_request() consumes the capture on the same service thread:
+ * it queues a physical GR submission, or falls back to the bridge (when
+ * enabled) and then the unchanged software release path. `offered` is the
+ * exactly-once latch: a capture is consumed at most once, and a rewind replay
+ * never re-captures because the QMD gate skips replay passes. Service-thread
+ * only, like every other decode/pump state.
+ */
+#define SVC_GR_QMD_WORDS	64
+#define SVC_GR_INLINE_MAX	8
+#define SVC_GR_INLINE_BYTES	64
+#define SVC_GR_POINTER_MAX	8
+#define SVC_GR_PAGES_MAX	16
+#define SVC_GR_QMD_WORD_NONE	0xffffffffu
+/* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */
+#define SVC_GR_REL_REDUCTION	0x1u
+#define SVC_GR_REL_ONE_WORD	0x2u
+#define SVC_GR_REL_ENABLE	0x4u
+
+struct svc_gr_inline {
+	u64 dst_va;
+	u32 len;
+	u8 data[SVC_GR_INLINE_BYTES];
+};
+
+/* One resolved launch page carried in the request/backend hand-off. */
+struct svc_gr_page {
+	u64 va;		/* guest VA, page aligned */
+	u64 pa;		/* guest PA, page aligned (0 = unresolved) */
+	u8 ap;		/* SVC_APERTURE_* of pa, 0 when pa = 0 */
+	u32 qmd_word;	/* QMD pointer pair word index, or NONE */
+};
+
+struct svc_gr_cand {
+	bool valid;
+	bool offered;		/* consumed by a request or a fallback */
+	u32 h_object;
+	u64 push_va;
+	u32 push_len;
+	u64 push_hash;
+	u32 qmd[SVC_GR_QMD_WORDS];
+	u64 qmd_dev_va;
+	u32 n_inline;
+	struct svc_gr_inline inline_recs[SVC_GR_INLINE_MAX];
+	/* Decoded by svc_gr_qmd_decode(); mirrors gsp_cuda_bridge.c:745-759. */
+	u64 prog_va;
+	u64 param_va;
+	u32 param_size;
+	u64 rel_va;
+	u32 rel_payload;
+	u32 rel_flags;		/* bit0 reduction, bit1 one-word, bit2 enable */
+	u32 n_pointer;
+	u64 pointer_va[SVC_GR_POINTER_MAX];
+	u32 pointer_word[SVC_GR_POINTER_MAX];	/* QMD word of the pair */
+};
+
+static struct svc_gr_cand svc_gr_cand;
+
+static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS,
+	      "the sema GR QMD word count must match the backend ABI");
+static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE,
+	      "the sema GR inline capture must fit the backend ABI");
+static_assert(sizeof(struct svc_gr_inline) == 80 &&
+	      offsetof(struct svc_gr_inline, data) == 12,
+	      "the sema GR inline record must stay layout-compatible with the backend");
+static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) &&
+	      offsetof(struct svc_gr_inline, dst_va) ==
+		      offsetof(struct gsp_bridge_inline, dst_va) &&
+	      offsetof(struct svc_gr_inline, len) ==
+		      offsetof(struct gsp_bridge_inline, len) &&
+	      offsetof(struct svc_gr_inline, data) ==
+		      offsetof(struct gsp_bridge_inline, data),
+	      "the sema GR inline record must be castable to the CUDA bridge record");
+static_assert(sizeof(struct svc_gr_inline) ==
+		      sizeof(struct gsp_phys_gr_inline) &&
+	      offsetof(struct svc_gr_inline, dst_va) ==
+		      offsetof(struct gsp_phys_gr_inline, dst_va) &&
+	      offsetof(struct svc_gr_inline, len) ==
+		      offsetof(struct gsp_phys_gr_inline, len) &&
+	      offsetof(struct svc_gr_inline, data) ==
+		      offsetof(struct gsp_phys_gr_inline, data),
+	      "the sema GR inline record must be castable to the backend record");
+static_assert(SVC_GR_PAGES_MAX == GSP_PHYS_GR_MAX_PAGES,
+	      "the sema GR page list must match the backend ABI");
+static_assert(sizeof(struct svc_gr_page) == 24,
+	      "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes");
+
+static u64 svc_push_hash(const u8 *p, u32 len);
+
+/* QMD field decode: the exact shape the host CUDA bridge uses. */
+static void svc_gr_qmd_decode(struct svc_gr_cand *c)
+{
+	const u32 *q = c->qmd;
+	u32 i;
+
+	c->rel_va = q[23] | ((u64)(q[24] & 0xff) << 32);
+	c->rel_payload = q[25];
+	c->rel_flags = 0;
+	if ((q[24] >> 26) & 1u)
+		c->rel_flags |= SVC_GR_REL_REDUCTION;
+	if ((q[24] >> 31) & 1u)
+		c->rel_flags |= SVC_GR_REL_ONE_WORD;
+	if ((q[4] >> 10) & 1u)
+		c->rel_flags |= SVC_GR_REL_ENABLE;
+	c->param_va = q[32] | ((u64)(q[33] & 0x1ffff) << 32);
+	c->param_size = (q[33] >> 19) << 4;
+	c->prog_va = q[48] | ((u64)(q[49] & 0x1ffff) << 32);
+	c->n_pointer = 0;
+	/* Pointer-region candidates 34..47; 48/49 is the program pair. */
+	for (i = 0; i < SVC_GR_POINTER_MAX - 1; i++) {
+		u32 w = 34 + 2 * i;
+		u64 va = q[w] | ((u64)(q[w + 1] & 0x1ffff) << 32);
+
+		if (!va)
+			continue;
+		c->pointer_va[c->n_pointer] = va;
+		c->pointer_word[c->n_pointer] = w;
+		c->n_pointer++;
+	}
+}
+
+/* Capture one completed QMD as the current GR forwarding candidate. */
+static void svc_gr_cand_capture(struct svc_channel *ch,
+				const struct svc_push_state *st,
+				const struct gsp_bridge_inline *recs, u32 n_recs,
+				const u8 *push, u32 push_len, u64 push_va)
+{
+	u32 i, n;
+
+	memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
+	svc_gr_cand.valid = true;
+	svc_gr_cand.h_object = ch->h_object;
+	svc_gr_cand.push_va = push_va;
+	svc_gr_cand.push_len = push_len;
+	if (push && push_len)
+		svc_gr_cand.push_hash = svc_push_hash(push, push_len);
+	memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd));
+	svc_gr_cand.qmd_dev_va = st->qmd_dev_va;
+	n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX;
+	for (i = 0; i < n; i++) {
+		svc_gr_cand.inline_recs[i].dst_va = recs[i].dst_va;
+		svc_gr_cand.inline_recs[i].len =
+			recs[i].len < SVC_GR_INLINE_BYTES ?
+			recs[i].len : SVC_GR_INLINE_BYTES;
+		memcpy(svc_gr_cand.inline_recs[i].data, recs[i].data,
+		       svc_gr_cand.inline_recs[i].len);
+	}
+	svc_gr_cand.n_inline = n;
+	svc_gr_qmd_decode(&svc_gr_cand);
+	pr_info("sema-gr: launch captured h=0x%x push=0x%llx len=%u qmd=0x%llx prog=0x%llx param=0x%llx size=0x%x rel=0x%llx/%u flags=0x%x ptr=%u inline=%u\n",
+		ch->h_object, push_va, push_len, st->qmd_dev_va,
+		svc_gr_cand.prog_va, svc_gr_cand.param_va,
+		svc_gr_cand.param_size, svc_gr_cand.rel_va,
+		svc_gr_cand.rel_payload, svc_gr_cand.rel_flags,
+		svc_gr_cand.n_pointer, svc_gr_cand.n_inline);
+}
+
+/*
+ * Consume the current GR candidate without forwarding it: hand the launch to
+ * the host CUDA bridge when that path is enabled, then clear the candidate.
+ * Exactly-once via `offered`. Called on the service thread (capture/refusal)
+ * or the work item (a failed physical submission uses the request copy).
+ */
+static void svc_gr_bridge_fallback(struct nv_vgpu_device *vgpu)
+{
+	u32 n;
+
+	if (!svc_gr_cand.valid || svc_gr_cand.offered)
+		return;
+	svc_gr_cand.offered = true;
+	if (gsp_cuda_bridge_enabled()) {
+		n = svc_gr_cand.n_inline;
+		if (n > GSP_BRIDGE_INLINE_MAX)
+			n = GSP_BRIDGE_INLINE_MAX;
+		gsp_cuda_bridge_launch(vgpu, svc_gr_cand.h_object,
+				       svc_gr_cand.qmd,
+				       svc_gr_cand.qmd_dev_va,
+				       svc_gr_cand.push_va,
+				       (const struct gsp_bridge_inline *)
+					       svc_gr_cand.inline_recs, n);
+		svc.fwd_gr_bridge++;
+	}
+	svc_gr_cand.valid = false;
+}
+
+/*
+ * Should this decode pass's releases be recorded instead of applied inline?
+ * A captured CE copy defers under the existing CE deferral parameter; a
+ * captured GR launch defers whenever GR forwarding is armed (its channel
+ * release must land only on the physical submission's success/failure edge).
+ * Both still require the guest-memory path that makes any hardware
+ * submission possible. Service-thread only, like the captures.
+ */
+static bool svc_fwd_defer_arm(void)
+{
+	if (!gsp_phys_guest_mem_enabled())
+		return false;
+	if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled())
+		return true;
+	return svc_fwd_cand.valid &&
+	       gsp_phys_work_defer_release_enabled();
+}
+
+
+/*
  * P6b wave-1.5 deferred guest releases: every release group the current
  * decode pass decoded while a forwarding candidate was captured is recorded
  * here instead of being applied inline. The record carries exactly what the
@@ -4475,15 +4701,23 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload)
 
 /*
  * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD
- * to the host CUDA bridge. The run may cover the address registers and the
- * data registers as a single INC/NON_INC/ONE_INC method (observed: op=1
- * count=66 starting at 0x318), so every data word is mapped to its register
- * offset. Each completed QMD resets the accumulator, so a push may carry
- * more than one launch.
+ * to its consumer. With gsp_phys_gr_forward=0 the consumer is the host CUDA
+ * bridge, exactly as before; with the parameter on a trusted decode captures
+ * the QMD as the current GR forwarding candidate (svc_gr_cand_capture) and
+ * an untrusted sweep still goes to the bridge, so the new path never acts on
+ * a content-scanned page. The bridge is then only reached from a
+ * request-path fallback, so a forwarded launch is never also handed to the
+ * bridge and a refused/failed forward still falls back in order forward ->
+ * bridge -> software release. The run may cover the
+ * address registers and the data registers as a single INC/NON_INC/ONE_INC
+ * method (observed: op=1 count=66 starting at 0x318), so every data word is
+ * mapped to its register offset. Each completed QMD resets the accumulator,
+ * so a push may carry more than one launch.
  */
 static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			  struct svc_push_state *st, u32 addr, u32 count,
-			  const u8 *data)
+			  const u8 *data, const u8 *push, u32 push_len,
+			  bool trusted)
 {
 	u32 i;
 
@@ -4542,9 +4776,13 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			memcpy(recs[n].data, rec->data, rec->len);
 			n++;
 		}
-		gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd,
-				       st->qmd_dev_va, st->push_va,
-				       recs, n);
+		if (gsp_phys_gr_forward_enabled() && trusted)
+			svc_gr_cand_capture(ch, st, recs, n, push, push_len,
+					    st->push_va);
+		else
+			gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd,
+					       st->qmd_dev_va, st->push_va,
+					       recs, n);
 		st->qmd_n = 0;
 		st->qmd_bits = 0;
 		st->qmd_have = 0;
@@ -4554,7 +4792,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			      u32 len, u64 page_pa, bool allow_ops,
 			      struct svc_channel *ch, u32 entry_idx,
-			      u64 push_va, bool trusted, u64 *ops_done)
+			      u64 push_va, bool trusted, bool replay,
+			      u64 *ops_done)
 {
 	struct svc_push_state st;
 	/*
@@ -4595,6 +4834,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	 */
 	svc_fwd_cand.valid = false;
 	svc_fwd_cand.skipped = false;
+	/*
+	 * A fresh (non-replay) trusted decode pass invalidates any previous GR
+	 * capture: the pump consumes a capture immediately after the pass that
+	 * built it, so anything still here is stale. Replay/rescue decodes
+	 * must not wipe a capture the pump has not consumed yet.
+	 */
+	if (!replay && trusted && ch->engine_type == 0)
+		memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
 	svc_fwd_rel_n = 0;
 	svc_fwd_defer_armed = false;
 	svc.pages_decoded++;
@@ -4739,13 +4986,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		 * must fall through to the CE decoder.
 		 */
 		if (ch->engine_type == 0 && subch >= 1 &&
-		    gsp_cuda_bridge_enabled() &&
+		    (!gsp_phys_gr_forward_enabled() || !replay) &&
+		    (gsp_cuda_bridge_enabled() ||
+		     gsp_phys_gr_forward_enabled()) &&
 		    addr >= SVC_QMD_ADDR_A &&
 		    addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 &&
 		    (st.qmd_have ||
 		     (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) {
 			svc_qmd_accum(vgpu, ch, &st, addr, count,
-				      buf + off + 4);
+				      buf + off + 4, buf, len, trusted);
 			off += 4 + count * 4;
 			continue;
 		}
@@ -4871,9 +5120,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * The pre-apply bookkeeping above is unchanged;
 				 * only the apply moves. Overflow applies inline.
 				 */
-				if (gsp_phys_work_defer_release_enabled() &&
-				    svc_fwd_cand.valid &&
-				    gsp_phys_guest_mem_enabled()) {
+				if (svc_fwd_defer_arm()) {
 					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 						struct svc_fwd_rel *r =
 							&svc_fwd_rel[svc_fwd_rel_n++];
@@ -4988,9 +5235,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 * once the forwarded copy completes.
 					 * Overflow applies inline.
 					 */
-					if (gsp_phys_work_defer_release_enabled() &&
-					    svc_fwd_cand.valid &&
-					    gsp_phys_guest_mem_enabled()) {
+					if (svc_fwd_defer_arm()) {
 						if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 							struct svc_fwd_rel *r =
 								&svc_fwd_rel[svc_fwd_rel_n++];
@@ -5386,9 +5631,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * the record path so the pump still sees the
 				 * release as decoded. Overflow applies inline.
 				 */
-				if (gsp_phys_work_defer_release_enabled() &&
-				    svc_fwd_cand.valid &&
-				    gsp_phys_guest_mem_enabled()) {
+				if (svc_fwd_defer_arm()) {
 					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 						struct svc_fwd_rel *r =
 							&svc_fwd_rel[svc_fwd_rel_n++];
@@ -7730,6 +7973,32 @@ struct svc_fwd_req {
 	 * (valid=false) for every non-fault outcome.
 	 */
 	struct gsp_phys_fault_info fault;
+	/*
+	 * GR (compute) forwarding (gsp_phys_gr_forward=1 only): when gr is
+	 * set the request is a captured guest launch, not a CE copy. The work
+	 * item resolves each page entry's guest PA to an HPA, copies the
+	 * captured inline bytes into the backend's shadow pages and calls
+	 * gsp_phys_backend_guest_gr_submit(); gr_bridge_done is the
+	 * exactly-once latch for the work-item CUDA bridge fallback.
+	 */
+	bool gr;
+	bool gr_bridge_done;
+	bool gr_page_overflow;
+	u32 gr_h_object;
+	u64 gr_push_va;
+	u32 gr_push_len;
+	u64 gr_qmd_dev_va;
+	u32 gr_rel_flags;
+	u64 gr_rel_va;
+	u32 gr_rel_payload;
+	u64 gr_prog_va;
+	u64 gr_param_va;
+	u32 gr_param_size;
+	u32 gr_qmd[SVC_GR_QMD_WORDS];
+	u32 gr_n_inline;
+	struct svc_gr_inline gr_inline[SVC_GR_INLINE_MAX];
+	u32 gr_n_pages;
+	struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX];
 };
 
 static struct svc_fwd_req svc_fwd_req;
@@ -7910,6 +8179,13 @@ struct svc_fwd_hold {
 	 */
 	struct nv_vgpu_device *vgpu;
 	u32 ch_handle;
+	/*
+	 * The channel's engine type at engage time: the shim channels the
+	 * release-apply paths build must preserve it so a GR (engine_type 0)
+	 * release lands through the exact completion-counter slot, exactly as
+	 * the inline path would have. CE holds stay 1.
+	 */
+	u32 engine_type;
 	u64 gpfifo_va;
 	u32 entry_idx;
 	u8 nrel;
@@ -8080,6 +8356,7 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu,
 	svc_fwd_hold.cand = svc_fwd_cand;
 	svc_fwd_hold.vgpu = vgpu;
 	svc_fwd_hold.ch_handle = ch->h_object;
+	svc_fwd_hold.engine_type = ch->engine_type;
 	svc_fwd_hold.gpfifo_va = ch->gpfifo_va;
 	svc_fwd_hold.entry_idx = entry_idx;
 	svc_fwd_hold.nrel = n;
@@ -8277,7 +8554,7 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu,
 		memset(&shim, 0, sizeof(shim));
 		shim.valid = true;
 		shim.h_object = svc_fwd_hold.ch_handle;
-		shim.engine_type = 1;	/* holds are CE-only */
+		shim.engine_type = svc_fwd_hold.engine_type;
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 	}
@@ -8441,7 +8718,7 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu)
 		memset(&shim, 0, sizeof(shim));
 		shim.valid = true;
 		shim.h_object = svc_fwd_hold.ch_handle;
-		shim.engine_type = 1;	/* holds are CE-only */
+		shim.engine_type = svc_fwd_hold.engine_type;
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 	}
@@ -8493,7 +8770,7 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
 		memset(&shim, 0, sizeof(shim));
 		shim.valid = true;
 		shim.h_object = svc_fwd_hold.ch_handle;
-		shim.engine_type = 1;	/* holds are CE-only */
+		shim.engine_type = svc_fwd_hold.engine_type;
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 	}
@@ -8508,6 +8785,135 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
 	svc_fwd_hold_clear();
 }
 
+static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu);
+
+/*
+ * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The
+ * request already resolved every launch page's guest VA->PA on the service
+ * thread; here each PA is turned into an HPA (SYS through the guest-RAM
+ * backing, VID through the BAR1 shadow), the captured inline bytes are handed
+ * to the backend for its Tier-2 shadow pages and the whole launch is
+ * submitted through gsp_phys_backend_guest_gr_submit(). On any failure the
+ * host CUDA bridge is offered the launch once (the forward -> bridge ->
+ * software fallback order), then the pump's hold applies the recorded
+ * releases. Runs on the gsp-phys-work item with the same lock discipline as
+ * the CE branch.
+ */
+static void svc_fwd_gr_work(void)
+{
+	/*
+	 * Static: the ordered gsp-phys-work queue runs one item at a time and
+	 * the GR page array is ~0.4 KB; keeping it off the stack keeps the
+	 * frame under the kernel's warning bound (same reason as the ops array
+	 * in svc_decode_stream).
+	 */
+	static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX];
+	struct gsp_phys_gr_submit gs;
+	struct gsp_phys_backend_copy_result cp = { 0 };
+	u64 rel_hpa = 0;
+	u32 off = 0, i;
+	int hw_rc;
+
+	memset(&gs, 0, sizeof(gs));
+	memset(pages, 0, sizeof(pages));
+
+	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+		const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i];
+		u8 kind = (pg->ap == SVC_APERTURE_FB) ?
+			  SVC_APERTURE_FB : SVC_APERTURE_SYS;
+		u64 hpa = 0;
+
+		if (pg->pa) {
+			if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, pg->pa,
+						0, SEMA_PAGE_SIZE, &hpa,
+						&off) || off) {
+				hpa = 0;
+				svc.fwd_guest_resolve_fail++;
+			}
+		}
+		pages[i].guest_va = pg->va;
+		pages[i].hpa = hpa;
+		pages[i].qmd_word = pg->qmd_word;
+	}
+
+	if (svc_fwd_req.gr_rel_va &&
+	    (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ONE_WORD) &&
+	    !(svc_fwd_req.gr_rel_flags & SVC_GR_REL_REDUCTION) &&
+	    (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ENABLE)) {
+		u64 page = svc_fwd_req.gr_rel_va &
+			   ~(u64)(SEMA_PAGE_SIZE - 1);
+		u32 roff = (u32)(svc_fwd_req.gr_rel_va &
+				 (SEMA_PAGE_SIZE - 1));
+
+		if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, SVC_APERTURE_SYS, page,
+					0, SEMA_PAGE_SIZE, &rel_hpa, &off) ||
+		    off)
+			rel_hpa = 0;
+		/*
+		 * Forward-only: a release word already at/ahead of the payload
+		 * means the engine would author nothing this launch; point
+		 * release0 at the module semaphore instead so the host still
+		 * observes completion.
+		 */
+		if (rel_hpa &&
+		    READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >=
+			    svc_fwd_req.gr_rel_payload)
+			rel_hpa = 0;
+	}
+
+	gs.qmd = svc_fwd_req.gr_qmd;
+	gs.qmd_words = SVC_GR_QMD_WORDS;
+	gs.qmd_va = svc_fwd_req.gr_qmd_dev_va;
+	gs.pages = pages;
+	gs.n_pages = svc_fwd_req.gr_n_pages;
+	gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline;
+	gs.n_inl = svc_fwd_req.gr_n_inline;
+	if (gs.n_inl > GSP_PHYS_GR_MAX_INLINE)
+		gs.n_inl = GSP_PHYS_GR_MAX_INLINE;
+	gs.rel_hpa = rel_hpa;
+	gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1));
+	gs.rel_payload = svc_fwd_req.gr_rel_payload;
+	gs.sem_hpa = 0;
+	gs.sem_off = 0;
+	gs.sem_payload = 0;
+	gs.push_va = svc_fwd_req.gr_push_va;
+	gs.push_len = svc_fwd_req.gr_push_len;
+	gs.h_object = svc_fwd_req.gr_h_object;
+
+	hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp);
+	if (!hw_rc) {
+		svc.fwd_gr_ok++;
+	} else {
+		svc.fwd_gr_fallback++;
+		if (hw_rc == -ETIMEDOUT) {
+			svc.fwd_gr_timeout++;
+			if (cp.fault.valid)
+				svc.fwd_gr_fault++;
+		}
+		if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1)
+			pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n",
+				hw_rc, svc_fwd_req.gr_h_object,
+				svc_fwd_req.gr_push_va, svc.fwd_gr_req,
+				svc.fwd_gr_ok, svc.fwd_gr_fallback);
+		svc_gr_req_bridge_fallback(svc_fwd_req.vgpu);
+	}
+
+	svc_fwd_req.fault = cp.fault;
+	svc_fwd_req.hw_rc = hw_rc;
+	/*
+	 * done before INFLIGHT: a reuser that acquires the bit can never
+	 * overwrite the slot before this outcome is visible.
+	 */
+	smp_store_release(&svc_fwd_req.done, true);
+	clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+	{
+		struct task_struct *task = READ_ONCE(svc.task);
+
+		if (task)
+			wake_up_process(task);
+	}
+}
+
 static void svc_fwd_work_fn(struct work_struct *work)
 {
 	bool guest = svc_fwd_req.guest;
@@ -8517,6 +8923,12 @@ static void svc_fwd_work_fn(struct work_struct *work)
 	int ret;
 	int hw_rc = 0;
 
+	/* GR forwarding requests take their own branch and outcome shape. */
+	if (svc_fwd_req.gr) {
+		svc_fwd_gr_work();
+		return;
+	}
+
 	/*
 	 * P6b wave 1: try the guest hardware copy first when the pump marked
 	 * this request guest. P6b wave 1.5: when the request's releases are
@@ -8733,6 +9145,230 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req)
 }
 
 /*
+ * GR request page collection: append (dedupe) one launch page, resolving its
+ * guest VA->PA on the service thread now (the work item only does PA->HPA).
+ * An unresolved page keeps pa=0; whether that is fatal is decided by
+ * svc_gr_req_fill from its QMD pointer association and the captured inline
+ * bytes. Overflow is a failure (fail-closed), not a silent drop.
+ */
+static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 pa = 0;
+	u32 ap = 0;
+	u32 i;
+
+	if (!page)
+		return;
+	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+		if (svc_fwd_req.gr_pages[i].va != page)
+			continue;
+		if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE &&
+		    qmd_word != SVC_GR_QMD_WORD_NONE)
+			svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
+		return;
+	}
+	if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) {
+		svc_fwd_req.gr_page_overflow = true;
+		return;
+	}
+	i = svc_fwd_req.gr_n_pages++;
+	svc_fwd_req.gr_pages[i].va = page;
+	svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
+	if (svc_resolve_va(vgpu, page, &pa, &ap))
+		return;
+	svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+	svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap);
+}
+
+/* Does any captured inline write land in this page? */
+static bool svc_gr_page_inline_covers(u64 page)
+{
+	u32 i;
+
+	for (i = 0; i < svc_gr_cand.n_inline; i++) {
+		if ((svc_gr_cand.inline_recs[i].dst_va &
+		     ~(u64)(SEMA_PAGE_SIZE - 1)) == page)
+			return true;
+	}
+	return false;
+}
+
+/*
+ * Copy the current GR capture into the request slot (INFLIGHT is held). Every
+ * launch page is collected and resolved; a page with no resolved PA is only
+ * acceptable when a QMD pointer pair names it and captured inline bytes cover
+ * it (the backend's Tier 2 relocation). Returns 0 or an errno the caller
+ * falls back on.
+ */
+static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
+			   const struct svc_channel *ch)
+{
+	u32 i;
+
+	svc_fwd_req.done = false;
+	svc_fwd_req.hw_rc = 0;
+	memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault));
+	svc_fwd_req.guest = false;
+	svc_fwd_req.gr = true;
+	svc_fwd_req.gr_bridge_done = false;
+	svc_fwd_req.defer = true;
+	svc_fwd_req.relay = false;
+	svc_fwd_req.relay_index = 0;
+	svc_fwd_req.vgpu = vgpu;
+	svc_fwd_req.gr_h_object = ch->h_object;
+	svc_fwd_req.gr_push_va = svc_gr_cand.push_va;
+	svc_fwd_req.gr_push_len = svc_gr_cand.push_len;
+	svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va;
+	svc_fwd_req.gr_rel_flags = svc_gr_cand.rel_flags;
+	svc_fwd_req.gr_rel_va = svc_gr_cand.rel_va;
+	svc_fwd_req.gr_rel_payload = svc_gr_cand.rel_payload;
+	svc_fwd_req.gr_prog_va = svc_gr_cand.prog_va;
+	svc_fwd_req.gr_param_va = svc_gr_cand.param_va;
+	svc_fwd_req.gr_param_size = svc_gr_cand.param_size;
+	memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd,
+	       sizeof(svc_fwd_req.gr_qmd));
+	svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline;
+	if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX)
+		svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX;
+	memcpy(svc_fwd_req.gr_inline, svc_gr_cand.inline_recs,
+	       sizeof(svc_fwd_req.gr_inline));
+	svc_fwd_req.gr_n_pages = 0;
+	svc_fwd_req.gr_page_overflow = false;
+
+	if (svc_gr_cand.param_va)
+		svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32);
+	if (svc_gr_cand.prog_va)
+		svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48);
+	for (i = 0; i < svc_gr_cand.n_pointer; i++)
+		svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i],
+				 svc_gr_cand.pointer_word[i]);
+	for (i = 0; i < svc_gr_cand.n_inline; i++)
+		svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va,
+				 SVC_GR_QMD_WORD_NONE);
+	if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages)
+		return -E2BIG;
+	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+		const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i];
+
+		if (pg->pa)
+			continue;
+		if (pg->qmd_word == SVC_GR_QMD_WORD_NONE)
+			return -ENOENT;
+		if (!svc_gr_page_inline_covers(pg->va))
+			return -ENOENT;
+	}
+	return 0;
+}
+
+/*
+ * Exactly-once CUDA bridge fallback from the request copy (work item or a
+ * failed queue): hand the captured launch to the bridge when enabled. The
+ * latch is per request slot and the slot is single-inflight.
+ */
+static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu)
+{
+	u32 n;
+
+	if (!svc_fwd_req.gr || svc_fwd_req.gr_bridge_done)
+		return;
+	svc_fwd_req.gr_bridge_done = true;
+	if (!gsp_cuda_bridge_enabled())
+		return;
+	n = svc_fwd_req.gr_n_inline;
+	if (n > GSP_BRIDGE_INLINE_MAX)
+		n = GSP_BRIDGE_INLINE_MAX;
+	gsp_cuda_bridge_launch(vgpu, svc_fwd_req.gr_h_object,
+			       svc_fwd_req.gr_qmd, svc_fwd_req.gr_qmd_dev_va,
+			       svc_fwd_req.gr_push_va,
+			       (const struct gsp_bridge_inline *)
+				       svc_fwd_req.gr_inline, n);
+	svc.fwd_gr_bridge++;
+}
+
+/*
+ * GR request path (gsp_phys_gr_forward=1 only), called by svc_fwd_request()
+ * for engine_type==0 channels. Queues one physical submission when the GR
+ * backend is ready; otherwise consumes the capture through the bridge
+ * fallback. Reuses the CE INFLIGHT/paused/hold exclusion so GR and CE
+ * forwarding can never overlap; the caller engages the release hold on a
+ * true return. Service-thread only.
+ */
+static bool svc_gr_request(struct nv_vgpu_device *vgpu,
+			   const struct svc_channel *ch)
+{
+	int rc;
+
+	if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid)
+		return false;
+	svc.fwd_gr_req++;
+	if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) {
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_fallback++;
+		svc_gr_bridge_fallback(vgpu);
+		return false;
+	}
+	/*
+	 * Never reuse the request slot while a hold is active; the held
+	 * request's done/hw_rc are still being read. The capture falls back.
+	 */
+	if (svc_fwd_hold.active) {
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_fallback++;
+		svc_gr_bridge_fallback(vgpu);
+		return false;
+	}
+	mutex_lock(&svc_fwd_lock);
+	if (svc_fwd_paused ||
+	    test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) {
+		mutex_unlock(&svc_fwd_lock);
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_fallback++;
+		svc_gr_bridge_fallback(vgpu);
+		return false;
+	}
+	rc = svc_gr_req_fill(vgpu, ch);
+	if (rc) {
+		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+		mutex_unlock(&svc_fwd_lock);
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_fallback++;
+		if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1)
+			pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n",
+				rc, ch->h_object, svc_gr_cand.push_va);
+		svc_gr_bridge_fallback(vgpu);
+		return false;
+	}
+	/*
+	 * The candidate is consumed: the request copy owns it from here, so a
+	 * later capture/replay cannot re-offer it and the bridge is only
+	 * reached through the request's own latch.
+	 */
+	svc_gr_cand.offered = true;
+	svc_gr_cand.valid = false;
+	svc_fwd_requests++;
+	if (!svc_fwd_wq) {
+		svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0);
+		if (!svc_fwd_wq) {
+			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+			svc_fwd_dropped++;
+			mutex_unlock(&svc_fwd_lock);
+			svc_gr_req_bridge_fallback(vgpu);
+			return false;
+		}
+	}
+	if (!queue_work(svc_fwd_wq, &svc_fwd_work)) {
+		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+		svc_fwd_dropped++;
+		mutex_unlock(&svc_fwd_lock);
+		svc_gr_req_bridge_fallback(vgpu);
+		return false;
+	}
+	mutex_unlock(&svc_fwd_lock);
+	return true;
+}
+
+/*
  * Called only from the service thread (the pump). Returns true when a guest
  * (hardware-copy) request was queued; the caller then decides whether to hold
  * the entry for its deferred releases. All wave-1 gates are unchanged.
@@ -8741,7 +9377,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
 			    const struct svc_channel *ch)
 {
 	if (!ch->engine_type)
-		return false;
+		return svc_gr_request(vgpu, ch);
 	if (!gsp_phys_work_forward_enabled())
 		return false;
 	/* Classify this decode pass's candidate for the run summary. */
@@ -8845,7 +9481,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
 		return false;
 	}
 	mutex_unlock(&svc_fwd_lock);
-	return svc_fwd_req.guest;
+	return svc_fwd_req.guest || svc_fwd_req.gr;
 }
 
 /*
@@ -8998,6 +9634,27 @@ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched,
 }
 
 /*
+ * GR forwarding counters for the gsp-phys-state work_gr line (see
+ * gsp_sema_service.h). Same plain-read discipline as the stats above.
+ */
+void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback,
+				   u64 *refused, u64 *timeout, u64 *fault)
+{
+	if (req)
+		*req = svc.fwd_gr_req;
+	if (ok)
+		*ok = svc.fwd_gr_ok;
+	if (fallback)
+		*fallback = svc.fwd_gr_fallback;
+	if (refused)
+		*refused = svc.fwd_gr_refused;
+	if (timeout)
+		*timeout = svc.fwd_gr_timeout;
+	if (fault)
+		*fault = svc.fwd_gr_fault;
+}
+
+/*
  * P6c: true while this channel owns an active RELAY hold. The pump's
  * per-pass notifier is suppressed for such a channel so the hold's verified
  * success edge (or the next pass after a failure resolution) is the single
@@ -9479,6 +10136,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									  idx,
 									  va,
 									  injected,
+									  entry_replay,
 									  entry_replay ?
 									  NULL :
 									  &entry_done);
@@ -9684,10 +10342,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 
 									if (!svc_rescue_push_page(vgpu, ch, va, len) &&
 									    !svc_read_va_push(vgpu, ch, va, pb, len))
-										svc_decode_stream(vgpu, pb,
-												  len, 0, false,
-												  ch, idx, va, true,
-												  NULL);
+									svc_decode_stream(vgpu, pb,
+											  len, 0, false,
+											  ch, idx, va, true,
+											  true,
+											  NULL);
 									rel_seen =
 										(svc_rel_count != rel_n2);
 									/*
@@ -9713,6 +10372,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 											svc_decode_stream(vgpu, pb,
 													  len, 0, false,
 													  ch, idx, va, true,
+													  true,
 													  NULL);
 										rel_seen =
 											(svc_rel_count != rel_n2);
@@ -9948,6 +10608,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 											  SVC_ENTRY_NONE,
 											  va,
 											  false,
+											  false,
 											  NULL);
 									if (svc_rel_count != rel_n_before)
 										ch->last_rel_va =
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 22e9edc..4b04892 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -148,6 +148,17 @@ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched,
 				      u64 *late_release);
 
 /*
+ * GR (compute) forwarding counters for the gsp-phys-state work_gr line:
+ * captures offered to the request path (req), launches the physical GR
+ * channel completed (ok), offerings that fell back to the CUDA bridge /
+ * software release (fallback), gate refusals (refused), backend timeouts
+ * (timeout) and timeouts carrying a decoded engine fault (fault). All zero
+ * while gsp_phys_gr_forward=0. Plain reads, safe from the debugfs show path.
+ */
+void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback,
+				   u64 *refused, u64 *timeout, u64 *fault);
+
+/*
  * Discover and pin GR (compute) channel rings after the CE region of the
  * channel video-memory allocation. Set false to leave GR rings untranslated
  * (the pump then only runs the CE channels).
diff --git a/kernel/main.c b/kernel/main.c
index f662b94..258de41 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -174,6 +174,7 @@ static rm_vgpu_vfio_ops_t g_rm_vgpu_vfio_ops;
 static struct mdev_parent *g_mdev_parent = NULL;
 
 static bool gsp_phys_gpu_param;
+static bool gsp_phys_gr_forward_param;
 
 /* VFIO ops implementation */
 static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id);
@@ -2596,6 +2597,23 @@ bool gsp_phys_gpu_enabled(void)
 }
 
 /*
+ * GR (compute) forwarding: when set, a guest GR channel's decoded inline-QMD
+ * launch is captured by the sema service and submitted as real work on the
+ * ladder's physical GR channel instead of (or before) the host CUDA bridge.
+ * Default off: every path is inert and the bridge/software emulation stays
+ * byte-identical. The backend additionally refuses until a real GR
+ * submission has proved out at boot (boot_stage.rmcall_gr_exec_ok), so the
+ * parameter alone can never make the module pretend GR works.
+ */
+module_param_named(gsp_phys_gr_forward, gsp_phys_gr_forward_param, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_gr_forward, "Forward guest GR (compute) launches to the physical GR engine on the debugfs ladder channel (default 0; requires gsp_phys_gpu + a proven GR ladder submission)");
+
+bool gsp_phys_gr_forward_enabled(void)
+{
+	return gsp_phys_gr_forward_param;
+}
+
+/*
  * GR ring discovery: pin the compute channels' rings/USERDs, which follow the
  * CE rings in the channel video-memory allocation. Default on; set 0 to leave
  * GR channels untranslated when debugging push-execution side effects.