vgsp: guest GR forwarding backend and capture (gsp_phys_gr_forward, default off)gr-forwarding

Add a default-off guest GR (compute) forwarding path so a guest CUDA
launch submitted on its GR channel can be executed by the physical GR
engine on the ladder channel, with fail-closed fallback to the host CUDA
bridge and the unchanged software release path.

- New module parameter gsp_phys_gr_forward (default 0) and accessor.
- New GR execution gate: boot_stage.rmcall_gr_exec_ok, latched only after
  a real GR submission logs REAL GR SEM OK; the backend refuses while it
  is unset.
- Sema capture: a completed 64-word inline QMD on a trusted, non-replay
  GR decode becomes an svc_gr_cand (push identity, decoded release0,
  parameter/program/pointer fields, up to 8 captured inline-write
  records); the CUDA bridge stays authoritative while the parameter is
  off and is reached in order forward -> bridge -> software release.
- Backend: gsp_phys_backend_guest_gr_submit() maps pages at their own
  guest VAs through fresh Tier-1 small-PT pages/PDE installs or relocates
  inline-captured pages to ladder shadow pages (Tier 2), builds
  SET_OBJECT + the inline-QMD method run + REPORT_SEMAPHORE on the
  compute subchannel, reuses the shared GPFIFO/USERD/doorbell publish
  tail (extracted from the CE submit path, CE bytes unchanged) and polls
  the QMD release and/or module semaphore, draining the epoch-gated
  fault queue on timeout.
- Sema wiring: GR requests reuse the INFLIGHT/hold/deferral machinery so
  the guest channel release applies only on the submit's success/failure
  edge; hold shims preserve the channel engine type.
- Counters and a work_gr state line plus a stats accessor.
- Bounds are fixed: 16 pages, 8 inline records, 64 QMD words; every
  pointer/size checked, pool exhaustion and stale mappings fail closed.
- New ABI layouts pinned with static_asserts.

Compile-only change: no device test performed.

1 files changed, 804 insertions, 60 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index d5585e4..b26f2bd 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -609,6 +609,21 @@ bool gsp_phys_boot_requested(void)
 	((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
 
 /*
+ * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources,
+ * allocated lazily under gsp_phys_rm_lock and freed with the staging. The
+ * Tier-1 window pool holds small-PT pages installed with DMA_UPDATE_PDE_2 at
+ * the guest's own 2 MiB VA windows (one PDE per window); the mapper's
+ * conservative no-cached-line rule only allows a new PTE in a fetch line the
+ * current submission installed, and never rewrites a stale one. The Tier-2
+ * shadow chunk backs relocated parameter pages whose content was captured as
+ * inline writes. Both are deliberately outside rmcall_chunks so the COPY0
+ * stage-2 registration walk stays byte-identical.
+ */
+#define GSP_PHYS_RMCALL_GR_PT_PAGES	8u
+#define GSP_PHYS_RMCALL_GR_PT_ENTRIES	32u
+#define GSP_PHYS_RMCALL_GR_SHADOW_PAGES	4u
+
+/*
  * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
  * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
  * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
@@ -1418,6 +1433,29 @@ static struct gsp_phys_boot_stage {
 	 */
 	u64 rmcall_va_guest_pt;		/* guest scratch-PT VA window base */
 	u32 rmcall_guest_pt_next_line;	/* next never-used 128 B PTE line */
+	/*
+	 * GR forwarding (gsp_phys_gr_forward=1; inert otherwise). The exec
+	 * gate latches only after the stage-4 real GR submission logs
+	 * REAL GR SEM OK, so the forward path can never pretend to work while
+	 * the ladder itself cannot execute GR. The window pool and shadow
+	 * chunk are allocated lazily by the forwarding submit.
+	 */
+	bool rmcall_gr_exec_ok;		/* a real GR submission proved out */
+	u32 rmcall_gr_submit_epoch;	/* per-submission mapper epoch */
+	u32 rmcall_gr_pt_used;		/* Tier-1 window pool slots used */
+	struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES];
+	struct gsp_phys_boot_chunk rmcall_gr_shadow;	/* Tier-2 pages */
+	struct {
+		u32 pde_index;
+		u8 pt_slot;
+		u8 used;
+		u8 pte_count;
+		struct {
+			u32 idx;	/* small-PT index in the 2 MiB window */
+			u32 epoch;	/* submission that installed it */
+			u64 hpa;	/* the PTE's mapped host PA */
+		} pte[GSP_PHYS_RMCALL_GR_PT_ENTRIES];
+	} rmcall_gr_win[GSP_PHYS_RMCALL_GR_PT_PAGES];
 	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
 	struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
 	u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
@@ -2016,6 +2054,13 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_guest_selftest_ok = false;
 	boot_stage.rmcall_va_guest_pt = 0;
 	boot_stage.rmcall_guest_pt_next_line = 0;
+	boot_stage.rmcall_gr_exec_ok = false;
+	boot_stage.rmcall_gr_submit_epoch = 0;
+	boot_stage.rmcall_gr_pt_used = 0;
+	memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt));
+	memset(&boot_stage.rmcall_gr_shadow, 0,
+	       sizeof(boot_stage.rmcall_gr_shadow));
+	memset(boot_stage.rmcall_gr_win, 0, sizeof(boot_stage.rmcall_gr_win));
 	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
 		boot_stage.rmcall_mem_handles[i] = 0;
 		boot_stage.rmcall_mem_registered[i] = false;
@@ -9434,84 +9479,39 @@ struct gsp_phys_rmcall_ce_submit {
 };
 
 /*
- * One CE copy block on the ladder channel: clear the module semaphore, build
- * the methods for bytes into the module pushbuffer, publish the GPFIFO entry
- * and USERD GP_PUT and ring the doorbell. The caller must hold
- * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical
- * lock-across-poll; the P5 work API polls without the lock). Every guard of
- * the original stage-4 submission body is preserved here: server-memory
- * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the
- * token check. The semaphore is cleared before the doorbell so a stale release
- * from a previous run cannot be observed as this submission's completion.
- * P6c: sem_payload is the release value the method block carries; legacy
- * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is
- * still cleared even when the caller polls a guest page: the clear only
- * guards the module-sem poll path and is a harmless non-guest write.
- *
- * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release
- * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and
- * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call
- * site passes false, so the default COPY0 ladder emits the same bytes as
- * before.
+ * Shared publish tail of the ladder submissions: read USERD GP_PUT/GP_GET,
+ * encode and write the GPFIFO entry for the module pushbuffer VA, advance
+ * GP_PUT, snapshot the C3 fault epoch and ring the PF doorbell. Extracted
+ * verbatim from gsp_phys_rmcall_ce_submit_locked so the CE path and the GR
+ * launch path publish byte-identically; the CE caller's semaphore clear and
+ * method-block build happen before this call. The caller must hold
+ * gsp_phys_rm_lock.
  */
-static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
-					    u64 dst_va, u64 sem_va, u32 bytes,
-					    u32 sem_payload, bool gr,
-					    struct gsp_phys_rmcall_ce_submit *out)
+static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va,
+					  u32 words,
+					  struct gsp_phys_rmcall_ce_submit *out)
 {
-	struct gsp_phys_boot_chunk *pb =
-		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
 	struct gsp_phys_boot_chunk *ring =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO];
 	struct gsp_phys_boot_chunk *userd_c =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
+	struct gsp_phys_boot_chunk *pb =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
 	struct gsp_phys_boot_chunk *sem =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
 	struct gsp_phys_boot_chunk *src =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
 	struct gsp_phys_boot_chunk *dst =
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
-	resource_size_t bar0_len;
-	u32 *pb_va, *userd, *ring_va;
-	u32 words, gpput, gpget, slot, e0, e1;
+	u32 *ring_va, *userd;
+	u32 gpput, gpget, slot, e0, e1;
 	u32 runlist_id, ch_id, doorbell;
-	u64 pb_gpu_va;
 
-	if (boot_stage.rmcall_channel_server_mem)
-		return -EOPNOTSUPP;
-	if (!boot_stage.pdev)
-		return -EINVAL;
-	bar0_len = pci_resource_len(boot_stage.pdev, 0);
-	if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4)
-		return -EIO;
-	if (!bar0 || !boot_stage.rmcall_token || !bytes ||
-	    bytes > GSP_PHYS_RMCALL_COPY_SIZE)
-		return -EINVAL;
-	pb_gpu_va = boot_stage.rmcall_va_pb;
-	if (!pb_gpu_va || (pb_gpu_va & 3))
-		return -EIO;
-
-	pb_va = gsp_phys_chunk_page_va(pb, 0);
 	ring_va = gsp_phys_chunk_page_va(ring, 0);
 	userd = gsp_phys_chunk_page_va(userd_c, 0);
-	if (!pb_va || !ring_va || !userd ||
-	    !gsp_phys_chunk_page_va(sem, 0) ||
-	    !gsp_phys_chunk_page_va(src, 0) ||
-	    !gsp_phys_chunk_page_va(dst, 0))
+	if (!ring_va || !userd)
 		return -EIO;
 
-	/* A stale release from a previous submission must not be observed. */
-	WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0);
-	wmb();
-
-	if (gr)
-		words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va,
-							 sem_payload);
-	else
-		words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va,
-						      dst_va, bytes,
-						      sem_payload);
-
 	gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
 	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
 	if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) {
@@ -9590,6 +9590,87 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
 }
 
 /*
+ * One CE copy block on the ladder channel: clear the module semaphore, build
+ * the methods for bytes into the module pushbuffer, publish the GPFIFO entry
+ * and USERD GP_PUT and ring the doorbell. The caller must hold
+ * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical
+ * lock-across-poll; the P5 work API polls without the lock). Every guard of
+ * the original stage-4 submission body is preserved here: server-memory
+ * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the
+ * token check. The semaphore is cleared before the doorbell so a stale release
+ * from a previous run cannot be observed as this submission's completion.
+ * P6c: sem_payload is the release value the method block carries; legacy
+ * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is
+ * still cleared even when the caller polls a guest page: the clear only
+ * guards the module-sem poll path and is a harmless non-guest write.
+ *
+ * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release
+ * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and
+ * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call
+ * site passes false, so the default COPY0 ladder emits the same bytes as
+ * before.
+ */
+static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
+					    u64 dst_va, u64 sem_va, u32 bytes,
+					    u32 sem_payload, bool gr,
+					    struct gsp_phys_rmcall_ce_submit *out)
+{
+	struct gsp_phys_boot_chunk *pb =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
+	struct gsp_phys_boot_chunk *ring =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO];
+	struct gsp_phys_boot_chunk *userd_c =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
+	struct gsp_phys_boot_chunk *sem =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+	struct gsp_phys_boot_chunk *src =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
+	struct gsp_phys_boot_chunk *dst =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
+	resource_size_t bar0_len;
+	u32 *pb_va, *userd, *ring_va;
+	u32 words;
+	u64 pb_gpu_va;
+
+	if (boot_stage.rmcall_channel_server_mem)
+		return -EOPNOTSUPP;
+	if (!boot_stage.pdev)
+		return -EINVAL;
+	bar0_len = pci_resource_len(boot_stage.pdev, 0);
+	if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4)
+		return -EIO;
+	if (!bar0 || !boot_stage.rmcall_token || !bytes ||
+	    bytes > GSP_PHYS_RMCALL_COPY_SIZE)
+		return -EINVAL;
+	pb_gpu_va = boot_stage.rmcall_va_pb;
+	if (!pb_gpu_va || (pb_gpu_va & 3))
+		return -EIO;
+
+	pb_va = gsp_phys_chunk_page_va(pb, 0);
+	ring_va = gsp_phys_chunk_page_va(ring, 0);
+	userd = gsp_phys_chunk_page_va(userd_c, 0);
+	if (!pb_va || !ring_va || !userd ||
+	    !gsp_phys_chunk_page_va(sem, 0) ||
+	    !gsp_phys_chunk_page_va(src, 0) ||
+	    !gsp_phys_chunk_page_va(dst, 0))
+		return -EIO;
+
+	/* A stale release from a previous submission must not be observed. */
+	WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0);
+	wmb();
+
+	if (gr)
+		words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va,
+							 sem_payload);
+	else
+		words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va,
+						      dst_va, bytes,
+						      sem_payload);
+
+	return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out);
+}
+
+/*
  * H1 stage-4 GR verification: one GR semaphore release on the ladder channel,
  * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller
  * must hold gsp_phys_rm_lock (stage 4 keeps its historical lock-across-poll;
@@ -9660,6 +9741,12 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
 		poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget,
 		get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
 		sub.words, boot_stage.rmcall_token);
+	/*
+	 * Latch the GR execution gate: this is the only place a real GR
+	 * submission is proven on the ladder, and gsp_phys_backend_gr_ready()
+	 * refuses every forwarded launch until this is observed.
+	 */
+	boot_stage.rmcall_gr_exec_ok = true;
 	return 0;
 }
 
@@ -9769,6 +9856,36 @@ bool gsp_phys_backend_work_ready(void)
 }
 EXPORT_SYMBOL_GPL(gsp_phys_backend_work_ready);
 
+/*
+ * GR forwarding readiness: the parameter, a live client-memory GR ladder
+ * channel and the latched boot proof that a real GR submission executed
+ * (rmcall_gr_exec_ok, set only by a REAL GR SEM OK). This is deliberately
+ * separate from gsp_phys_backend_ladder_ready(), which refuses CE work on a
+ * GR0 ladder; a GR0 ladder is exactly the GR forwarding prerequisite. Every
+ * submit re-checks the same state under gsp_phys_rm_lock.
+ */
+bool gsp_phys_backend_gr_ready(void)
+{
+	if (!gsp_phys_gr_forward_enabled())
+		return false;
+	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY)
+		return false;
+	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
+	    boot_stage.rm_unload_submitted)
+		return false;
+	if (!gsp_phys_rmcall_engine_is_gr() || !boot_stage.rmcall_gr_exec_ok)
+		return false;
+	if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem ||
+	    !boot_stage.rmcall_token)
+		return false;
+	if (!boot_stage.rmcall_va_pb || !boot_stage.rmcall_va_gpfifo ||
+	    !boot_stage.rmcall_va_src || !boot_stage.rmcall_va_sem ||
+	    !boot_stage.rmcall_pdb_root)
+		return false;
+	return true;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_ready);
+
 bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va,
 				    u32 *bytes)
 {
@@ -10525,6 +10642,595 @@ int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
 }
 EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
 
+/* ------------------------------------------------------------------ */
+/* GR (compute) forwarding backend (gsp_phys_gr_forward)               */
+/* ------------------------------------------------------------------ */
+
+/*
+ * C5C0 inline-QMD method window: SET_INLINE_QMD_ADDRESS_A/B carry the
+ * shifted QMD device address and LOAD_INLINE_QMD_DATA the 64 QMD words; the
+ * guest emits the window as one INC run at 0x318 with count 66 (A, B, then 64
+ * words), and the launch is replayed in exactly that shape with the QMD words
+ * the sema service decoded (release0 and, under Tier 2, relocated pointers
+ * rewritten). The channel release is the same A..D block as the H1 GR sem
+ * verify, with a payload distinct from the CE/module sentinel.
+ */
+#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A	0x318u
+#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_B	0x31cu
+#define GSP_PHYS_RMCALL_GR_LOAD_INLINE_QMD_DATA	0x320u
+/* Header count: ADDRESS_A + ADDRESS_B + 64 QMD words. */
+#define GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS	66u
+#define GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD	GSP_PHYS_RMCALL_SEM_PAYLOAD
+#define GSP_PHYS_RMCALL_GR_SEM_PAYLOAD		0xc0ffee02u
+/* Method block bound: SET_OBJECT(2) + inline run(67) + REPORT_SEM(8). */
+#define GSP_PHYS_RMCALL_GR_METHOD_WORDS		96u
+
+static u64 gsp_phys_gr_submitted;
+static u64 gsp_phys_gr_ok;
+static u64 gsp_phys_gr_refused;
+static u64 gsp_phys_gr_timed_out;
+static u64 gsp_phys_gr_pool_exhausted;
+
+void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused,
+				  u64 *timed_out, u64 *pool_exhausted)
+{
+	if (submitted)
+		*submitted = gsp_phys_gr_submitted;
+	if (ok)
+		*ok = gsp_phys_gr_ok;
+	if (refused)
+		*refused = gsp_phys_gr_refused;
+	if (timed_out)
+		*timed_out = gsp_phys_gr_timed_out;
+	if (pool_exhausted)
+		*pool_exhausted = gsp_phys_gr_pool_exhausted;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_counters);
+
+/*
+ * GR launch method block: SET_OBJECT on the compute subchannel, the inline
+ * QMD run at its original shape and the channel REPORT_SEMAPHORE release.
+ * Returns the block size in dwords.
+ */
+static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const u32 *qmd,
+					   u64 qmd_va, u64 sem_va,
+					   u32 sem_payload)
+{
+	u32 *p = pb;
+
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT,
+				    GSP_PHYS_RMCALL_CLASS_COMPUTE);
+	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+	       (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) |
+	       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
+	       (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2);
+	*p++ = (u32)(qmd_va >> 40);
+	*p++ = (u32)((qmd_va >> 8) & 0xffffffffu);
+	memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
+	p += GSP_PHYS_GR_QMD_WORDS;
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A,
+				    (u32)(sem_va >> 32));
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B,
+				    (u32)sem_va);
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C,
+				    sem_payload);
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D,
+				    GSP_PHYS_RMCALL_GR_SEM_RELEASE_D);
+	return (u32)(p - pb);
+}
+
+/*
+ * Tier-1 window pool: find (or install) the small-PT page for one guest
+ * 2 MiB VA window through DMA_UPDATE_PDE_2. Installation is once per window
+ * per staging, so a later launch in the same window writes PTEs into the same
+ * server-installed PT page. Called with gsp_phys_rm_lock held.
+ */
+static int gsp_phys_rmcall_gr_win_slot_locked(void __iomem *bar0,
+					      u32 pde_index, u32 *slot_out)
+{
+	struct gsp_phys_rmcall_update_pde2_params p;
+	struct gsp_phys_boot_chunk *pt;
+	u32 rpc_result = 0, status = 0;
+	u32 i, slot;
+	int ret;
+
+	for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) {
+		if (boot_stage.rmcall_gr_win[i].used &&
+		    boot_stage.rmcall_gr_win[i].pde_index == pde_index) {
+			*slot_out = i;
+			return 0;
+		}
+	}
+	if (boot_stage.rmcall_gr_pt_used >= GSP_PHYS_RMCALL_GR_PT_PAGES)
+		return -ENOSPC;
+	slot = boot_stage.rmcall_gr_pt_used;
+	pt = &boot_stage.rmcall_gr_pt[slot];
+	if (!pt->pages) {
+		ret = gsp_phys_chunk_alloc(boot_stage.pdev, pt, 1u);
+		if (ret)
+			return ret;
+		gsp_phys_chunk_zero(pt);
+	}
+	memset(&p, 0, sizeof(p));
+	p.pde_index = cpu_to_le32(pde_index);
+	p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
+	p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0));
+	p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
+	p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
+	p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
+	p.pde_buffer = 0;
+	p.sub_device_id = 0;
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
+			GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
+			&p, sizeof(p), sizeof(p), &p, sizeof(p),
+			&rpc_result, &status, "GR forward PDE_UPDATE");
+	if (ret) {
+		pr_err("vgsp-gr: forward PDE install refused pdeIndex=%u (window VA 0x%llx) status=0x%x rpc_result=0x%x ret=%d\n",
+		       pde_index, (u64)pde_index << 21, status, rpc_result, ret);
+		return ret;
+	}
+	boot_stage.rmcall_gr_win[slot].pde_index = pde_index;
+	boot_stage.rmcall_gr_win[slot].pt_slot = (u8)slot;
+	boot_stage.rmcall_gr_win[slot].used = true;
+	boot_stage.rmcall_gr_win[slot].pte_count = 0;
+	boot_stage.rmcall_gr_pt_used = slot + 1;
+	pr_info("vgsp-gr: forward PDE installed pdeIndex=%u slot=%u ptPa=0x%llx status=0x%x\n",
+		pde_index, slot, (u64)gsp_phys_chunk_page_dma(pt, 0), status);
+	*slot_out = slot;
+	return 0;
+}
+
+/*
+ * Tier-1 map: write one guest page's HPA into the PT page of its own 2 MiB
+ * VA window at the guest VA itself, so the QMD and every pointer inside the
+ * mapped pages resolve without rewriting. The GMMU caches 128 B PTE fetch
+ * lines with no invalidate, so the mapper never rewrites an installed PTE
+ * (a different HPA for the same VA is refused rather than made stale) and
+ * only adds a new PTE to a fetch line this same submission already touched.
+ * Every refusal is fail-closed: the caller drops to Tier 2 or refuses the
+ * launch and the sema service falls back. Called with gsp_phys_rm_lock held.
+ */
+static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va,
+					    u64 hpa, u32 epoch)
+{
+	u64 page = guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+	u32 pde_index = (u32)(page >> 21);
+	u32 pte_idx = (u32)((page >> 12) & 0x1ffu);
+	u32 line = pte_idx / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES;
+	struct gsp_phys_boot_chunk *pt;
+	u8 *pt_va;
+	u32 slot, i;
+	int ret;
+
+	ret = gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot);
+	if (ret)
+		return ret;
+	for (i = 0; i < boot_stage.rmcall_gr_win[slot].pte_count; i++) {
+		if (boot_stage.rmcall_gr_win[slot].pte[i].idx == pte_idx) {
+			if (boot_stage.rmcall_gr_win[slot].pte[i].hpa == hpa)
+				return 0;	/* already mapped, no write */
+			return -EALREADY;	/* stale: never rewrite */
+		}
+		if (boot_stage.rmcall_gr_win[slot].pte[i].epoch != epoch &&
+		    boot_stage.rmcall_gr_win[slot].pte[i].idx /
+			    GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES == line)
+			return -EBUSY;	/* fetch line may be cached */
+	}
+	if (boot_stage.rmcall_gr_win[slot].pte_count >=
+	    GSP_PHYS_RMCALL_GR_PT_ENTRIES)
+		return -ENOSPC;
+	pt = &boot_stage.rmcall_gr_pt[boot_stage.rmcall_gr_win[slot].pt_slot];
+	pt_va = gsp_phys_chunk_page_va(pt, 0);
+	if (!pt_va)
+		return -EIO;
+	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
+			   pt_va + (size_t)pte_idx * 8);
+	dma_wmb();
+	boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx;
+	boot_stage.rmcall_gr_win[slot].pte[i].epoch = epoch;
+	boot_stage.rmcall_gr_win[slot].pte[i].hpa = hpa;
+	boot_stage.rmcall_gr_win[slot].pte_count++;
+	return 0;
+}
+
+/*
+ * Tier-2 map: consume one never-reused 128 B fetch line from the guest
+ * scratch PT pool (PTE0 only) for a ladder-owned shadow page and return the
+ * line's base VA. The shadow chunk is allocated lazily; page slot `slot`
+ * supplies the backing page. Called with gsp_phys_rm_lock held.
+ */
+static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa,
+						u64 *va_out)
+{
+	struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow;
+	struct gsp_phys_boot_chunk *pt;
+	u32 line = boot_stage.rmcall_guest_pt_next_line;
+	u32 page, line_in_page;
+	u64 va;
+	u8 *pt_va;
+	int ret;
+
+	if (!sh->pages) {
+		ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
+					   GSP_PHYS_RMCALL_GR_SHADOW_PAGES);
+		if (ret)
+			return ret;
+		gsp_phys_chunk_zero(sh);
+	}
+	if (slot >= sh->pages)
+		return -ENOSPC;
+	if (!boot_stage.rmcall_va_guest_pt)
+		return -EIO;
+	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
+		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
+		return -ENOSPC;
+	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
+	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
+	pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
+	if (!pt->pages)
+		return -EIO;
+	pt_va = gsp_phys_chunk_page_va(pt, 0);
+	if (!pt_va)
+		return -EIO;
+	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
+			   pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
+					    line_in_page) * 8);
+	dma_wmb();
+	boot_stage.rmcall_guest_pt_next_line = line + 1;
+	va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
+	     ((u64)line_in_page << 16);
+	*va_out = va;
+	return 0;
+}
+
+/* GR forward fault observation is armed by the GR parameters, not the CE
+ * ones (gsp_phys_fault_relay_active() requires gsp_phys_work_forward). */
+static bool gsp_phys_gr_fault_active(void)
+{
+	return gsp_phys_fault_relay_enabled() &&
+	       gsp_phys_gr_forward_enabled() &&
+	       gsp_phys_guest_mem_enabled();
+}
+
+/*
+ * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The
+ * caller has resolved the launch's pages; every mapping is fresh or proven
+ * unchanged, the QMD release0 and the channel release are authored either by
+ * the engine into ladder-mapped shadow pages or into the module semaphore,
+ * and the GPFIFO/USERD/doorbell tail is the shared publish helper. The
+ * release poll runs without gsp_phys_rm_lock; a timeout drains the status
+ * queue (epoch-gated) into out->fault. Every refusal moves the GR counters
+ * and returns an errno the sema service falls back on.
+ */
+int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
+				     struct gsp_phys_backend_copy_result *out)
+{
+	struct gsp_phys_boot_chunk *sem =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+	struct gsp_phys_boot_chunk *pb =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
+	void __iomem *bar0;
+	struct gsp_phys_rmcall_ce_submit sub;
+	struct gsp_phys_fault_info fault = { };
+	/*
+	 * Static scratch: the only caller is the single-inflight sema work
+	 * item (ordered gsp-phys-work queue), so no concurrent submission can
+	 * touch these. Keeping the 96-dword method block and the 64-word QMD
+	 * copy off the stack keeps the frame under the warning bound.
+	 */
+	static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS];
+	static u32 qmd[GSP_PHYS_GR_QMD_WORDS];
+	unsigned long start = jiffies;
+	unsigned long deadline;
+	u8 *sem_va;
+	u32 epoch, words, i, n_shadow = 0;
+	u32 want_qmd = 0, want_ch = 0;
+	bool qmd_rel = false, ch_rel = false;
+	u64 qmd_rel_hpa = 0, ch_rel_hpa = 0;
+	u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4;
+	u32 qmd_rel_off = 0, ch_rel_off = 0;
+	u32 poll = 0, w0 = 0, w1 = 0;
+	int ret = 0;
+
+	if (out)
+		memset(out, 0, sizeof(*out));
+	if (!req || !req->qmd ||
+	    req->qmd_words != GSP_PHYS_GR_QMD_WORDS ||
+	    req->n_pages > GSP_PHYS_GR_MAX_PAGES ||
+	    req->n_inl > GSP_PHYS_GR_MAX_INLINE) {
+		gsp_phys_gr_refused++;
+		return -EINVAL;
+	}
+	if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() ||
+	    gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) {
+		gsp_phys_gr_refused++;
+		ret = -EAGAIN;
+		goto out;
+	}
+	bar0 = gsp_phys_gpu_bar0();
+	if (!bar0) {
+		gsp_phys_gr_refused++;
+		ret = -EAGAIN;
+		goto out;
+	}
+	sem_va = gsp_phys_chunk_page_va(sem, 0);
+	if (!sem_va || !gsp_phys_chunk_page_va(pb, 0)) {
+		gsp_phys_gr_refused++;
+		ret = -EIO;
+		goto out;
+	}
+
+	gsp_phys_gr_submitted++;
+	mutex_lock(&gsp_phys_rm_lock);
+	if (!gsp_phys_backend_gr_ready()) {
+		mutex_unlock(&gsp_phys_rm_lock);
+		gsp_phys_gr_refused++;
+		ret = -EAGAIN;
+		goto out;
+	}
+	epoch = ++boot_stage.rmcall_gr_submit_epoch;
+	if (!epoch)
+		epoch = ++boot_stage.rmcall_gr_submit_epoch;
+	memcpy(qmd, req->qmd, sizeof(qmd));
+
+	/*
+	 * QMD release0: when the launch has a one-word absolute release and
+	 * the caller resolved the guest release page, author it there through
+	 * a fresh shadow line so the guest observes its own QMD release; else
+	 * point release0 at the module semaphore so the host observes engine
+	 * completion. A reduction release is never redirected (its arithmetic
+	 * would change).
+	 */
+	if (((qmd[4] >> 10) & 1)) {
+		u32 red_en = (qmd[24] >> 26) & 1;
+		u32 one_word = (qmd[24] >> 31) & 1;
+
+		if (req->rel_hpa && one_word && !red_en &&
+		    n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
+			u64 sh_va = 0;
+
+			ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow,
+								   req->rel_hpa,
+								   &sh_va);
+			if (!ret) {
+				n_shadow++;
+				qmd_rel_hpa = req->rel_hpa;
+				qmd_rel_off = req->rel_off;
+				want_qmd = req->rel_payload;
+				qmd_rel = true;
+				qmd[23] = (u32)(sh_va + req->rel_off);
+				qmd[24] = (qmd[24] & 0xffffff00u) |
+					  ((u32)((sh_va + req->rel_off) >> 32) &
+					   0xffu);
+				qmd[25] = req->rel_payload;
+			} else {
+				ret = 0;
+			}
+		}
+		if (!qmd_rel) {
+			qmd[23] = (u32)GSP_PHYS_RMCALL_VA_SEM;
+			qmd[24] = (qmd[24] & 0xffffff00u) |
+				  ((u32)(GSP_PHYS_RMCALL_VA_SEM >> 32) &
+				   0xffu);
+			qmd[25] = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD;
+			want_qmd = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD;
+			qmd_rel = true;
+		}
+	}
+
+	/*
+	 * Pages: Tier 1 maps the resolved HPA at the guest VA; a page without
+	 * a usable Tier-1 mapping falls to Tier 2 when captured inline bytes
+	 * cover it and a QMD pointer pair names it (the pair is rewritten to
+	 * the shadow VA); anything else refuses the launch.
+	 */
+	for (i = 0; i < req->n_pages; i++) {
+		const struct gsp_phys_gr_page *pg = &req->pages[i];
+		u64 pg_va, sh_va = 0;
+		bool mapped = false;
+
+		if (!pg->guest_va) {
+			mutex_unlock(&gsp_phys_rm_lock);
+			gsp_phys_gr_refused++;
+			ret = -EINVAL;
+			goto out;
+		}
+		pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+		if (pg->hpa) {
+			ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va,
+							       pg->hpa, epoch);
+			if (!ret)
+				mapped = true;
+			else if (ret == -ENOSPC)
+				gsp_phys_gr_pool_exhausted++;
+		}
+		if (!mapped && pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE &&
+		    pg->qmd_word + 1 < GSP_PHYS_GR_QMD_WORDS) {
+			struct gsp_phys_boot_chunk *sh =
+				&boot_stage.rmcall_gr_shadow;
+			u8 *sh_page;
+			u32 j, copied = 0;
+
+			if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
+				gsp_phys_gr_pool_exhausted++;
+				continue;
+			}
+			if (!sh->pages) {
+				ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
+							   GSP_PHYS_RMCALL_GR_SHADOW_PAGES);
+				if (ret) {
+					gsp_phys_gr_pool_exhausted++;
+					continue;
+				}
+				gsp_phys_chunk_zero(sh);
+			}
+			sh_page = gsp_phys_chunk_page_va(sh, n_shadow);
+			if (!sh_page)
+				continue;
+			memset(sh_page, 0, GSP_PHYS_PAGE_SIZE);
+			for (j = 0; j < req->n_inl; j++) {
+				const struct gsp_phys_gr_inline *inl =
+					&req->inl[j];
+				u32 len;
+
+				if ((inl->dst_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))
+				    != pg_va)
+					continue;
+				len = inl->len;
+				if (len > GSP_PHYS_GR_INLINE_BYTES)
+					len = GSP_PHYS_GR_INLINE_BYTES;
+				if ((u64)(inl->dst_va &
+					  (GSP_PHYS_PAGE_SIZE - 1)) + len >
+				    GSP_PHYS_PAGE_SIZE)
+					continue;
+				memcpy(sh_page + (inl->dst_va &
+						  (GSP_PHYS_PAGE_SIZE - 1)),
+				       inl->data, len);
+				copied++;
+			}
+			if (copied) {
+				ret = gsp_phys_rmcall_gr_map_shadow_locked(
+					n_shadow,
+					(u64)gsp_phys_chunk_page_dma(sh,
+								      n_shadow),
+					&sh_va);
+				if (!ret) {
+					u64 ptr_va = sh_va +
+						(pg->guest_va &
+						 (GSP_PHYS_PAGE_SIZE - 1));
+
+					n_shadow++;
+					qmd[pg->qmd_word] = (u32)ptr_va;
+					qmd[pg->qmd_word + 1] =
+						(qmd[pg->qmd_word + 1] &
+						 ~0x1ffffu) |
+						((u32)(ptr_va >> 32) & 0x1ffffu);
+					mapped = true;
+				}
+			}
+		}
+		if (!mapped) {
+			mutex_unlock(&gsp_phys_rm_lock);
+			gsp_phys_gr_refused++;
+			ret = -EOPNOTSUPP;
+			pr_info("vgsp-gr: launch refused: page va=0x%llx hpa=0x%llx qmd_word=%u has no Tier-1 mapping and no captured inline data (pages=%u)\n",
+				pg->guest_va, pg->hpa, pg->qmd_word,
+				req->n_pages);
+			goto out;
+		}
+	}
+
+	/*
+	 * Channel release: author it into the guest semaphore page when the
+	 * caller resolved one, else into the module semaphore's second word.
+	 */
+	if (req->sem_hpa && req->sem_payload &&
+	    n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
+		u64 sh_va = 0;
+
+		ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow,
+							   req->sem_hpa, &sh_va);
+		if (!ret) {
+			n_shadow++;
+			ch_rel_hpa = req->sem_hpa;
+			ch_rel_off = req->sem_off;
+			want_ch = req->sem_payload;
+			ch_rel = true;
+			ch_sem_va = sh_va + req->sem_off;
+		}
+	}
+	if (!ch_rel) {
+		want_ch = GSP_PHYS_RMCALL_GR_SEM_PAYLOAD;
+		ch_rel = true;
+	}
+
+	/* A stale release from a previous submission must not be observed. */
+	WRITE_ONCE(*(u32 *)sem_va, 0);
+	WRITE_ONCE(*(u32 *)(sem_va + 4), 0);
+	wmb();
+
+	words = gsp_phys_rmcall_build_gr_launch(methods, qmd,
+						GSP_PHYS_RMCALL_VA_SRC,
+						ch_sem_va, want_ch);
+	memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
+
+	ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb,
+					     words, &sub);
+	mutex_unlock(&gsp_phys_rm_lock);
+	if (ret) {
+		gsp_phys_gr_refused++;
+		pr_err("vgsp-gr: launch submit refused (%d) pages=%u words=%u\n",
+		       ret, req->n_pages, words);
+		goto out;
+	}
+
+	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
+		req->h_object, req->push_va, req->push_len, req->n_pages,
+		n_shadow, req->qmd_va, sub.slot, words, boot_stage.rmcall_token,
+		qmd_rel, want_ch, GSP_PHYS_RMCALL_GUEST_POLL_MS);
+
+	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS);
+	for (;;) {
+		if (qmd_rel) {
+			if (qmd_rel_hpa)
+				w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) +
+							qmd_rel_off));
+			else
+				w0 = READ_ONCE(*(u32 *)sem_va);
+		}
+		if (ch_rel_hpa)
+			w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) +
+						ch_rel_off));
+		else
+			w1 = READ_ONCE(*(u32 *)(sem_va + 4));
+		if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch)
+			break;
+		if (time_after_eq(jiffies, deadline)) {
+			gsp_phys_gr_timed_out++;
+			pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n",
+			       GSP_PHYS_RMCALL_GUEST_POLL_MS, w0, want_qmd, w1,
+			       want_ch, req->h_object, req->push_va);
+			if (gsp_phys_gr_fault_active()) {
+				struct gsp_phys_fault_info fi;
+				u32 fep = 0;
+
+				if (!gsp_phys_status_drain_fault(&fi, &fep) &&
+				    fi.valid &&
+				    fi.function ==
+				    GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) {
+					if (fep > READ_ONCE(gsp_phys_fault_submit_epoch)) {
+						fault = fi;
+						gsp_phys_fault_drained++;
+					} else {
+						gsp_phys_fault_stale++;
+					}
+				}
+			}
+			poll = w1;
+			ret = -ETIMEDOUT;
+			goto out;
+		}
+		msleep(GSP_PHYS_RMCALL_POLL_MS);
+	}
+
+	gsp_phys_gr_ok++;
+	poll = w1;
+	pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n",
+		req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1,
+		jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok);
+	ret = 0;
+
+out:
+	if (out) {
+		out->sem_value = poll;
+		out->elapsed_ms = jiffies_to_msecs(jiffies - start);
+		out->rc = ret;
+		out->fault = fault;
+	}
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_gr_submit);
+
 /*
  * P6a trigger variant: identical to the exported forwarding call except that
  * the gsp_phys_work_forward parameter gate is dropped (every other guard and
@@ -11296,6 +12002,9 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx);
 	for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++)
 		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx_pt[i]);
+	for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++)
+		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]);
+	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs);
@@ -11529,6 +12238,30 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 	}
 	{
 		/*
+		 * GR (compute) forwarding: the sema service's capture/offer/
+		 * fallback counters and the booter backend's submit/refusal
+		 * counters plus the parameter and exec-gate state. All zero
+		 * while gsp_phys_gr_forward=0.
+		 */
+		u64 gr_req = 0, gr_ok = 0, gr_fallback = 0, gr_refused = 0;
+		u64 gr_timeout = 0, gr_fault = 0;
+		u64 b_sub = 0, b_ok = 0, b_refused = 0, b_timeout = 0;
+		u64 b_pool = 0;
+
+		gsp_sema_service_fwd_gr_stats(&gr_req, &gr_ok, &gr_fallback,
+					      &gr_refused, &gr_timeout,
+					      &gr_fault);
+		gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused,
+					     &b_timeout, &b_pool);
+		seq_printf(m,
+			   "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u\n",
+			   gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout,
+			   gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool,
+			   gsp_phys_gr_forward_enabled(),
+			   boot_stage.rmcall_gr_exec_ok);
+	}
+	{
+		/*
 		 * P6c C3 fault observation/relay: the booter's 0x1006
 		 * observation counters (seen/stale/drained/wait_unmatched,
 		 * epoch) and the sema service's delivery counters
@@ -12835,6 +13568,17 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX +
 		      (u64)GSP_PHYS_RMCALL_GR_CTX_PT_MAX * (2ULL << 20) <=
 		      GSP_PHYS_RMCALL_VAS_SIZE,
 	      "the GR ctx window must stay below the VAS vaSize limit");
+static_assert(sizeof(struct gsp_phys_gr_page) == 24,
+	      "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes");
+static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 &&
+	      sizeof(struct gsp_phys_gr_inline) == 80,
+	      "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes");
+static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <=
+	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
+	      "the GR launch method block (SET_OBJECT + inline-QMD run + REPORT_SEMAPHORE) must fit the bound");
+static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <=
+	      GSP_PHYS_RMCALL_COPY_SIZE,
+	      "the GR launch method block must fit one module pushbuffer page");
 
 /*
  * GspSystemInfo wire ABI, pinned against the reference header compile