vgsp: guest GR forwarding backend and capture (gsp_phys_gr_forward, default off)

Add a default-off guest GR (compute) forwarding path so a guest CUDA
launch submitted on its GR channel can be executed by the physical GR
engine on the ladder channel, with fail-closed fallback to the host CUDA
bridge and the unchanged software release path.

- New module parameter gsp_phys_gr_forward (default 0) and accessor.
- New GR execution gate: boot_stage.rmcall_gr_exec_ok, latched only after
  a real GR submission logs REAL GR SEM OK; the backend refuses while it
  is unset.
- Sema capture: a completed 64-word inline QMD on a trusted, non-replay
  GR decode becomes an svc_gr_cand (push identity, decoded release0,
  parameter/program/pointer fields, up to 8 captured inline-write
  records); the CUDA bridge stays authoritative while the parameter is
  off and is reached in order forward -> bridge -> software release.
- Backend: gsp_phys_backend_guest_gr_submit() maps pages at their own
  guest VAs through fresh Tier-1 small-PT pages/PDE installs or relocates
  inline-captured pages to ladder shadow pages (Tier 2), builds
  SET_OBJECT + the inline-QMD method run + REPORT_SEMAPHORE on the
  compute subchannel, reuses the shared GPFIFO/USERD/doorbell publish
  tail (extracted from the CE submit path, CE bytes unchanged) and polls
  the QMD release and/or module semaphore, draining the epoch-gated
  fault queue on timeout.
- Sema wiring: GR requests reuse the INFLIGHT/hold/deferral machinery so
  the guest channel release applies only on the submit's success/failure
  edge; hold shims preserve the channel engine type.
- Counters and a work_gr state line plus a stats accessor.
- Bounds are fixed: 16 pages, 8 inline records, 64 QMD words; every
  pointer/size checked, pool exhaustion and stale mappings fail closed.
- New ABI layouts pinned with static_asserts.

Compile-only change: no device test performed.

(cherry picked from commit ce40bd872ea05776156a68afcc6d4571a0bbcad2)

1 files changed, 691 insertions, 30 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index f897ce0..2b7fa80 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -611,6 +611,23 @@ struct sema_svc {
 	u64			fwd_fault_delivered;
 	u64			fwd_fault_no_event;
 	u64			fwd_fault_late_release;
+	/*
+	 * GR (compute) forwarding (gsp_phys_gr_forward=1 only): candidate
+	 * launches offered to the request path (fwd_gr_req), launches the
+	 * physical GR channel completed (fwd_gr_ok), offerings that fell back
+	 * to the host CUDA bridge / software release (fwd_gr_fallback), gate
+	 * refusals (fwd_gr_refused), backend timeouts (fwd_gr_timeout),
+	 * timeouts carrying a decoded engine fault (fwd_gr_fault) and CUDA
+	 * bridge launch calls made by a fallback (fwd_gr_bridge). All zero
+	 * while the parameter is off.
+	 */
+	u64			fwd_gr_req;
+	u64			fwd_gr_ok;
+	u64			fwd_gr_fallback;
+	u64			fwd_gr_refused;
+	u64			fwd_gr_timeout;
+	u64			fwd_gr_fault;
+	u64			fwd_gr_bridge;
 
 	/* Distinct stalled VA tracking for rate-limited logging. */
 #define SVC_STALLED_MAX		256
@@ -3154,6 +3171,215 @@ struct svc_fwd_cand {
 static struct svc_fwd_cand svc_fwd_cand;
 
 /*
+ * GR (compute) forwarding candidate (gsp_phys_gr_forward=1 only). When a GR
+ * channel decodes a completed 64-word inline QMD, the launch is captured here
+ * instead of being handed straight to the host CUDA bridge: the QMD words, its
+ * device VA, the push identity, the newest GRCE inline-write records and the
+ * QMD fields the bridge decodes (release0, parameter bank, program, pointer
+ * pairs). svc_fwd_request() consumes the capture on the same service thread:
+ * it queues a physical GR submission, or falls back to the bridge (when
+ * enabled) and then the unchanged software release path. `offered` is the
+ * exactly-once latch: a capture is consumed at most once, and a rewind replay
+ * never re-captures because the QMD gate skips replay passes. Service-thread
+ * only, like every other decode/pump state.
+ */
+#define SVC_GR_QMD_WORDS	64
+#define SVC_GR_INLINE_MAX	8
+#define SVC_GR_INLINE_BYTES	64
+#define SVC_GR_POINTER_MAX	8
+#define SVC_GR_PAGES_MAX	16
+#define SVC_GR_QMD_WORD_NONE	0xffffffffu
+/* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */
+#define SVC_GR_REL_REDUCTION	0x1u
+#define SVC_GR_REL_ONE_WORD	0x2u
+#define SVC_GR_REL_ENABLE	0x4u
+
+struct svc_gr_inline {
+	u64 dst_va;
+	u32 len;
+	u8 data[SVC_GR_INLINE_BYTES];
+};
+
+/* One resolved launch page carried in the request/backend hand-off. */
+struct svc_gr_page {
+	u64 va;		/* guest VA, page aligned */
+	u64 pa;		/* guest PA, page aligned (0 = unresolved) */
+	u8 ap;		/* SVC_APERTURE_* of pa, 0 when pa = 0 */
+	u32 qmd_word;	/* QMD pointer pair word index, or NONE */
+};
+
+struct svc_gr_cand {
+	bool valid;
+	bool offered;		/* consumed by a request or a fallback */
+	u32 h_object;
+	u64 push_va;
+	u32 push_len;
+	u64 push_hash;
+	u32 qmd[SVC_GR_QMD_WORDS];
+	u64 qmd_dev_va;
+	u32 n_inline;
+	struct svc_gr_inline inline_recs[SVC_GR_INLINE_MAX];
+	/* Decoded by svc_gr_qmd_decode(); mirrors gsp_cuda_bridge.c:745-759. */
+	u64 prog_va;
+	u64 param_va;
+	u32 param_size;
+	u64 rel_va;
+	u32 rel_payload;
+	u32 rel_flags;		/* bit0 reduction, bit1 one-word, bit2 enable */
+	u32 n_pointer;
+	u64 pointer_va[SVC_GR_POINTER_MAX];
+	u32 pointer_word[SVC_GR_POINTER_MAX];	/* QMD word of the pair */
+};
+
+static struct svc_gr_cand svc_gr_cand;
+
+static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS,
+	      "the sema GR QMD word count must match the backend ABI");
+static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE,
+	      "the sema GR inline capture must fit the backend ABI");
+static_assert(sizeof(struct svc_gr_inline) == 80 &&
+	      offsetof(struct svc_gr_inline, data) == 12,
+	      "the sema GR inline record must stay layout-compatible with the backend");
+static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) &&
+	      offsetof(struct svc_gr_inline, dst_va) ==
+		      offsetof(struct gsp_bridge_inline, dst_va) &&
+	      offsetof(struct svc_gr_inline, len) ==
+		      offsetof(struct gsp_bridge_inline, len) &&
+	      offsetof(struct svc_gr_inline, data) ==
+		      offsetof(struct gsp_bridge_inline, data),
+	      "the sema GR inline record must be castable to the CUDA bridge record");
+static_assert(sizeof(struct svc_gr_inline) ==
+		      sizeof(struct gsp_phys_gr_inline) &&
+	      offsetof(struct svc_gr_inline, dst_va) ==
+		      offsetof(struct gsp_phys_gr_inline, dst_va) &&
+	      offsetof(struct svc_gr_inline, len) ==
+		      offsetof(struct gsp_phys_gr_inline, len) &&
+	      offsetof(struct svc_gr_inline, data) ==
+		      offsetof(struct gsp_phys_gr_inline, data),
+	      "the sema GR inline record must be castable to the backend record");
+static_assert(SVC_GR_PAGES_MAX == GSP_PHYS_GR_MAX_PAGES,
+	      "the sema GR page list must match the backend ABI");
+static_assert(sizeof(struct svc_gr_page) == 24,
+	      "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes");
+
+static u64 svc_push_hash(const u8 *p, u32 len);
+
+/* QMD field decode: the exact shape the host CUDA bridge uses. */
+static void svc_gr_qmd_decode(struct svc_gr_cand *c)
+{
+	const u32 *q = c->qmd;
+	u32 i;
+
+	c->rel_va = q[23] | ((u64)(q[24] & 0xff) << 32);
+	c->rel_payload = q[25];
+	c->rel_flags = 0;
+	if ((q[24] >> 26) & 1u)
+		c->rel_flags |= SVC_GR_REL_REDUCTION;
+	if ((q[24] >> 31) & 1u)
+		c->rel_flags |= SVC_GR_REL_ONE_WORD;
+	if ((q[4] >> 10) & 1u)
+		c->rel_flags |= SVC_GR_REL_ENABLE;
+	c->param_va = q[32] | ((u64)(q[33] & 0x1ffff) << 32);
+	c->param_size = (q[33] >> 19) << 4;
+	c->prog_va = q[48] | ((u64)(q[49] & 0x1ffff) << 32);
+	c->n_pointer = 0;
+	/* Pointer-region candidates 34..47; 48/49 is the program pair. */
+	for (i = 0; i < SVC_GR_POINTER_MAX - 1; i++) {
+		u32 w = 34 + 2 * i;
+		u64 va = q[w] | ((u64)(q[w + 1] & 0x1ffff) << 32);
+
+		if (!va)
+			continue;
+		c->pointer_va[c->n_pointer] = va;
+		c->pointer_word[c->n_pointer] = w;
+		c->n_pointer++;
+	}
+}
+
+/* Capture one completed QMD as the current GR forwarding candidate. */
+static void svc_gr_cand_capture(struct svc_channel *ch,
+				const struct svc_push_state *st,
+				const struct gsp_bridge_inline *recs, u32 n_recs,
+				const u8 *push, u32 push_len, u64 push_va)
+{
+	u32 i, n;
+
+	memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
+	svc_gr_cand.valid = true;
+	svc_gr_cand.h_object = ch->h_object;
+	svc_gr_cand.push_va = push_va;
+	svc_gr_cand.push_len = push_len;
+	if (push && push_len)
+		svc_gr_cand.push_hash = svc_push_hash(push, push_len);
+	memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd));
+	svc_gr_cand.qmd_dev_va = st->qmd_dev_va;
+	n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX;
+	for (i = 0; i < n; i++) {
+		svc_gr_cand.inline_recs[i].dst_va = recs[i].dst_va;
+		svc_gr_cand.inline_recs[i].len =
+			recs[i].len < SVC_GR_INLINE_BYTES ?
+			recs[i].len : SVC_GR_INLINE_BYTES;
+		memcpy(svc_gr_cand.inline_recs[i].data, recs[i].data,
+		       svc_gr_cand.inline_recs[i].len);
+	}
+	svc_gr_cand.n_inline = n;
+	svc_gr_qmd_decode(&svc_gr_cand);
+	pr_info("sema-gr: launch captured h=0x%x push=0x%llx len=%u qmd=0x%llx prog=0x%llx param=0x%llx size=0x%x rel=0x%llx/%u flags=0x%x ptr=%u inline=%u\n",
+		ch->h_object, push_va, push_len, st->qmd_dev_va,
+		svc_gr_cand.prog_va, svc_gr_cand.param_va,
+		svc_gr_cand.param_size, svc_gr_cand.rel_va,
+		svc_gr_cand.rel_payload, svc_gr_cand.rel_flags,
+		svc_gr_cand.n_pointer, svc_gr_cand.n_inline);
+}
+
+/*
+ * Consume the current GR candidate without forwarding it: hand the launch to
+ * the host CUDA bridge when that path is enabled, then clear the candidate.
+ * Exactly-once via `offered`. Called on the service thread (capture/refusal)
+ * or the work item (a failed physical submission uses the request copy).
+ */
+static void svc_gr_bridge_fallback(struct nv_vgpu_device *vgpu)
+{
+	u32 n;
+
+	if (!svc_gr_cand.valid || svc_gr_cand.offered)
+		return;
+	svc_gr_cand.offered = true;
+	if (gsp_cuda_bridge_enabled()) {
+		n = svc_gr_cand.n_inline;
+		if (n > GSP_BRIDGE_INLINE_MAX)
+			n = GSP_BRIDGE_INLINE_MAX;
+		gsp_cuda_bridge_launch(vgpu, svc_gr_cand.h_object,
+				       svc_gr_cand.qmd,
+				       svc_gr_cand.qmd_dev_va,
+				       svc_gr_cand.push_va,
+				       (const struct gsp_bridge_inline *)
+					       svc_gr_cand.inline_recs, n);
+		svc.fwd_gr_bridge++;
+	}
+	svc_gr_cand.valid = false;
+}
+
+/*
+ * Should this decode pass's releases be recorded instead of applied inline?
+ * A captured CE copy defers under the existing CE deferral parameter; a
+ * captured GR launch defers whenever GR forwarding is armed (its channel
+ * release must land only on the physical submission's success/failure edge).
+ * Both still require the guest-memory path that makes any hardware
+ * submission possible. Service-thread only, like the captures.
+ */
+static bool svc_fwd_defer_arm(void)
+{
+	if (!gsp_phys_guest_mem_enabled())
+		return false;
+	if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled())
+		return true;
+	return svc_fwd_cand.valid &&
+	       gsp_phys_work_defer_release_enabled();
+}
+
+
+/*
  * P6b wave-1.5 deferred guest releases: every release group the current
  * decode pass decoded while a forwarding candidate was captured is recorded
  * here instead of being applied inline. The record carries exactly what the
@@ -4475,15 +4701,23 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload)
 
 /*
  * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD
- * to the host CUDA bridge. The run may cover the address registers and the
- * data registers as a single INC/NON_INC/ONE_INC method (observed: op=1
- * count=66 starting at 0x318), so every data word is mapped to its register
- * offset. Each completed QMD resets the accumulator, so a push may carry
- * more than one launch.
+ * to its consumer. With gsp_phys_gr_forward=0 the consumer is the host CUDA
+ * bridge, exactly as before; with the parameter on a trusted decode captures
+ * the QMD as the current GR forwarding candidate (svc_gr_cand_capture) and
+ * an untrusted sweep still goes to the bridge, so the new path never acts on
+ * a content-scanned page. The bridge is then only reached from a
+ * request-path fallback, so a forwarded launch is never also handed to the
+ * bridge and a refused/failed forward still falls back in order forward ->
+ * bridge -> software release. The run may cover the
+ * address registers and the data registers as a single INC/NON_INC/ONE_INC
+ * method (observed: op=1 count=66 starting at 0x318), so every data word is
+ * mapped to its register offset. Each completed QMD resets the accumulator,
+ * so a push may carry more than one launch.
  */
 static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			  struct svc_push_state *st, u32 addr, u32 count,
-			  const u8 *data)
+			  const u8 *data, const u8 *push, u32 push_len,
+			  bool trusted)
 {
 	u32 i;
 
@@ -4542,9 +4776,13 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			memcpy(recs[n].data, rec->data, rec->len);
 			n++;
 		}
-		gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd,
-				       st->qmd_dev_va, st->push_va,
-				       recs, n);
+		if (gsp_phys_gr_forward_enabled() && trusted)
+			svc_gr_cand_capture(ch, st, recs, n, push, push_len,
+					    st->push_va);
+		else
+			gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd,
+					       st->qmd_dev_va, st->push_va,
+					       recs, n);
 		st->qmd_n = 0;
 		st->qmd_bits = 0;
 		st->qmd_have = 0;
@@ -4554,7 +4792,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			      u32 len, u64 page_pa, bool allow_ops,
 			      struct svc_channel *ch, u32 entry_idx,
-			      u64 push_va, bool trusted, u64 *ops_done)
+			      u64 push_va, bool trusted, bool replay,
+			      u64 *ops_done)
 {
 	struct svc_push_state st;
 	/*
@@ -4595,6 +4834,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	 */
 	svc_fwd_cand.valid = false;
 	svc_fwd_cand.skipped = false;
+	/*
+	 * A fresh (non-replay) trusted decode pass invalidates any previous GR
+	 * capture: the pump consumes a capture immediately after the pass that
+	 * built it, so anything still here is stale. Replay/rescue decodes
+	 * must not wipe a capture the pump has not consumed yet.
+	 */
+	if (!replay && trusted && ch->engine_type == 0)
+		memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
 	svc_fwd_rel_n = 0;
 	svc_fwd_defer_armed = false;
 	svc.pages_decoded++;
@@ -4739,13 +4986,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		 * must fall through to the CE decoder.
 		 */
 		if (ch->engine_type == 0 && subch >= 1 &&
-		    gsp_cuda_bridge_enabled() &&
+		    (!gsp_phys_gr_forward_enabled() || !replay) &&
+		    (gsp_cuda_bridge_enabled() ||
+		     gsp_phys_gr_forward_enabled()) &&
 		    addr >= SVC_QMD_ADDR_A &&
 		    addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 &&
 		    (st.qmd_have ||
 		     (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) {
 			svc_qmd_accum(vgpu, ch, &st, addr, count,
-				      buf + off + 4);
+				      buf + off + 4, buf, len, trusted);
 			off += 4 + count * 4;
 			continue;
 		}
@@ -4871,9 +5120,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * The pre-apply bookkeeping above is unchanged;
 				 * only the apply moves. Overflow applies inline.
 				 */
-				if (gsp_phys_work_defer_release_enabled() &&
-				    svc_fwd_cand.valid &&
-				    gsp_phys_guest_mem_enabled()) {
+				if (svc_fwd_defer_arm()) {
 					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 						struct svc_fwd_rel *r =
 							&svc_fwd_rel[svc_fwd_rel_n++];
@@ -4988,9 +5235,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 * once the forwarded copy completes.
 					 * Overflow applies inline.
 					 */
-					if (gsp_phys_work_defer_release_enabled() &&
-					    svc_fwd_cand.valid &&
-					    gsp_phys_guest_mem_enabled()) {
+					if (svc_fwd_defer_arm()) {
 						if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 							struct svc_fwd_rel *r =
 								&svc_fwd_rel[svc_fwd_rel_n++];
@@ -5386,9 +5631,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * the record path so the pump still sees the
 				 * release as decoded. Overflow applies inline.
 				 */
-				if (gsp_phys_work_defer_release_enabled() &&
-				    svc_fwd_cand.valid &&
-				    gsp_phys_guest_mem_enabled()) {
+				if (svc_fwd_defer_arm()) {
 					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 						struct svc_fwd_rel *r =
 							&svc_fwd_rel[svc_fwd_rel_n++];
@@ -7730,6 +7973,32 @@ struct svc_fwd_req {
 	 * (valid=false) for every non-fault outcome.
 	 */
 	struct gsp_phys_fault_info fault;
+	/*
+	 * GR (compute) forwarding (gsp_phys_gr_forward=1 only): when gr is
+	 * set the request is a captured guest launch, not a CE copy. The work
+	 * item resolves each page entry's guest PA to an HPA, copies the
+	 * captured inline bytes into the backend's shadow pages and calls
+	 * gsp_phys_backend_guest_gr_submit(); gr_bridge_done is the
+	 * exactly-once latch for the work-item CUDA bridge fallback.
+	 */
+	bool gr;
+	bool gr_bridge_done;
+	bool gr_page_overflow;
+	u32 gr_h_object;
+	u64 gr_push_va;
+	u32 gr_push_len;
+	u64 gr_qmd_dev_va;
+	u32 gr_rel_flags;
+	u64 gr_rel_va;
+	u32 gr_rel_payload;
+	u64 gr_prog_va;
+	u64 gr_param_va;
+	u32 gr_param_size;
+	u32 gr_qmd[SVC_GR_QMD_WORDS];
+	u32 gr_n_inline;
+	struct svc_gr_inline gr_inline[SVC_GR_INLINE_MAX];
+	u32 gr_n_pages;
+	struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX];
 };
 
 static struct svc_fwd_req svc_fwd_req;
@@ -7910,6 +8179,13 @@ struct svc_fwd_hold {
 	 */
 	struct nv_vgpu_device *vgpu;
 	u32 ch_handle;
+	/*
+	 * The channel's engine type at engage time: the shim channels the
+	 * release-apply paths build must preserve it so a GR (engine_type 0)
+	 * release lands through the exact completion-counter slot, exactly as
+	 * the inline path would have. CE holds stay 1.
+	 */
+	u32 engine_type;
 	u64 gpfifo_va;
 	u32 entry_idx;
 	u8 nrel;
@@ -8080,6 +8356,7 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu,
 	svc_fwd_hold.cand = svc_fwd_cand;
 	svc_fwd_hold.vgpu = vgpu;
 	svc_fwd_hold.ch_handle = ch->h_object;
+	svc_fwd_hold.engine_type = ch->engine_type;
 	svc_fwd_hold.gpfifo_va = ch->gpfifo_va;
 	svc_fwd_hold.entry_idx = entry_idx;
 	svc_fwd_hold.nrel = n;
@@ -8277,7 +8554,7 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu,
 		memset(&shim, 0, sizeof(shim));
 		shim.valid = true;
 		shim.h_object = svc_fwd_hold.ch_handle;
-		shim.engine_type = 1;	/* holds are CE-only */
+		shim.engine_type = svc_fwd_hold.engine_type;
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 	}
@@ -8441,7 +8718,7 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu)
 		memset(&shim, 0, sizeof(shim));
 		shim.valid = true;
 		shim.h_object = svc_fwd_hold.ch_handle;
-		shim.engine_type = 1;	/* holds are CE-only */
+		shim.engine_type = svc_fwd_hold.engine_type;
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 	}
@@ -8493,7 +8770,7 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
 		memset(&shim, 0, sizeof(shim));
 		shim.valid = true;
 		shim.h_object = svc_fwd_hold.ch_handle;
-		shim.engine_type = 1;	/* holds are CE-only */
+		shim.engine_type = svc_fwd_hold.engine_type;
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 	}
@@ -8508,6 +8785,135 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
 	svc_fwd_hold_clear();
 }
 
+static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu);
+
+/*
+ * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The
+ * request already resolved every launch page's guest VA->PA on the service
+ * thread; here each PA is turned into an HPA (SYS through the guest-RAM
+ * backing, VID through the BAR1 shadow), the captured inline bytes are handed
+ * to the backend for its Tier-2 shadow pages and the whole launch is
+ * submitted through gsp_phys_backend_guest_gr_submit(). On any failure the
+ * host CUDA bridge is offered the launch once (the forward -> bridge ->
+ * software fallback order), then the pump's hold applies the recorded
+ * releases. Runs on the gsp-phys-work item with the same lock discipline as
+ * the CE branch.
+ */
+static void svc_fwd_gr_work(void)
+{
+	/*
+	 * Static: the ordered gsp-phys-work queue runs one item at a time and
+	 * the GR page array is ~0.4 KB; keeping it off the stack keeps the
+	 * frame under the kernel's warning bound (same reason as the ops array
+	 * in svc_decode_stream).
+	 */
+	static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX];
+	struct gsp_phys_gr_submit gs;
+	struct gsp_phys_backend_copy_result cp = { 0 };
+	u64 rel_hpa = 0;
+	u32 off = 0, i;
+	int hw_rc;
+
+	memset(&gs, 0, sizeof(gs));
+	memset(pages, 0, sizeof(pages));
+
+	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+		const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i];
+		u8 kind = (pg->ap == SVC_APERTURE_FB) ?
+			  SVC_APERTURE_FB : SVC_APERTURE_SYS;
+		u64 hpa = 0;
+
+		if (pg->pa) {
+			if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, pg->pa,
+						0, SEMA_PAGE_SIZE, &hpa,
+						&off) || off) {
+				hpa = 0;
+				svc.fwd_guest_resolve_fail++;
+			}
+		}
+		pages[i].guest_va = pg->va;
+		pages[i].hpa = hpa;
+		pages[i].qmd_word = pg->qmd_word;
+	}
+
+	if (svc_fwd_req.gr_rel_va &&
+	    (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ONE_WORD) &&
+	    !(svc_fwd_req.gr_rel_flags & SVC_GR_REL_REDUCTION) &&
+	    (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ENABLE)) {
+		u64 page = svc_fwd_req.gr_rel_va &
+			   ~(u64)(SEMA_PAGE_SIZE - 1);
+		u32 roff = (u32)(svc_fwd_req.gr_rel_va &
+				 (SEMA_PAGE_SIZE - 1));
+
+		if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, SVC_APERTURE_SYS, page,
+					0, SEMA_PAGE_SIZE, &rel_hpa, &off) ||
+		    off)
+			rel_hpa = 0;
+		/*
+		 * Forward-only: a release word already at/ahead of the payload
+		 * means the engine would author nothing this launch; point
+		 * release0 at the module semaphore instead so the host still
+		 * observes completion.
+		 */
+		if (rel_hpa &&
+		    READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >=
+			    svc_fwd_req.gr_rel_payload)
+			rel_hpa = 0;
+	}
+
+	gs.qmd = svc_fwd_req.gr_qmd;
+	gs.qmd_words = SVC_GR_QMD_WORDS;
+	gs.qmd_va = svc_fwd_req.gr_qmd_dev_va;
+	gs.pages = pages;
+	gs.n_pages = svc_fwd_req.gr_n_pages;
+	gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline;
+	gs.n_inl = svc_fwd_req.gr_n_inline;
+	if (gs.n_inl > GSP_PHYS_GR_MAX_INLINE)
+		gs.n_inl = GSP_PHYS_GR_MAX_INLINE;
+	gs.rel_hpa = rel_hpa;
+	gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1));
+	gs.rel_payload = svc_fwd_req.gr_rel_payload;
+	gs.sem_hpa = 0;
+	gs.sem_off = 0;
+	gs.sem_payload = 0;
+	gs.push_va = svc_fwd_req.gr_push_va;
+	gs.push_len = svc_fwd_req.gr_push_len;
+	gs.h_object = svc_fwd_req.gr_h_object;
+
+	hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp);
+	if (!hw_rc) {
+		svc.fwd_gr_ok++;
+	} else {
+		svc.fwd_gr_fallback++;
+		if (hw_rc == -ETIMEDOUT) {
+			svc.fwd_gr_timeout++;
+			if (cp.fault.valid)
+				svc.fwd_gr_fault++;
+		}
+		if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1)
+			pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n",
+				hw_rc, svc_fwd_req.gr_h_object,
+				svc_fwd_req.gr_push_va, svc.fwd_gr_req,
+				svc.fwd_gr_ok, svc.fwd_gr_fallback);
+		svc_gr_req_bridge_fallback(svc_fwd_req.vgpu);
+	}
+
+	svc_fwd_req.fault = cp.fault;
+	svc_fwd_req.hw_rc = hw_rc;
+	/*
+	 * done before INFLIGHT: a reuser that acquires the bit can never
+	 * overwrite the slot before this outcome is visible.
+	 */
+	smp_store_release(&svc_fwd_req.done, true);
+	clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+	{
+		struct task_struct *task = READ_ONCE(svc.task);
+
+		if (task)
+			wake_up_process(task);
+	}
+}
+
 static void svc_fwd_work_fn(struct work_struct *work)
 {
 	bool guest = svc_fwd_req.guest;
@@ -8517,6 +8923,12 @@ static void svc_fwd_work_fn(struct work_struct *work)
 	int ret;
 	int hw_rc = 0;
 
+	/* GR forwarding requests take their own branch and outcome shape. */
+	if (svc_fwd_req.gr) {
+		svc_fwd_gr_work();
+		return;
+	}
+
 	/*
 	 * P6b wave 1: try the guest hardware copy first when the pump marked
 	 * this request guest. P6b wave 1.5: when the request's releases are
@@ -8733,6 +9145,230 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req)
 }
 
 /*
+ * GR request page collection: append (dedupe) one launch page, resolving its
+ * guest VA->PA on the service thread now (the work item only does PA->HPA).
+ * An unresolved page keeps pa=0; whether that is fatal is decided by
+ * svc_gr_req_fill from its QMD pointer association and the captured inline
+ * bytes. Overflow is a failure (fail-closed), not a silent drop.
+ */
+static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 pa = 0;
+	u32 ap = 0;
+	u32 i;
+
+	if (!page)
+		return;
+	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+		if (svc_fwd_req.gr_pages[i].va != page)
+			continue;
+		if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE &&
+		    qmd_word != SVC_GR_QMD_WORD_NONE)
+			svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
+		return;
+	}
+	if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) {
+		svc_fwd_req.gr_page_overflow = true;
+		return;
+	}
+	i = svc_fwd_req.gr_n_pages++;
+	svc_fwd_req.gr_pages[i].va = page;
+	svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
+	if (svc_resolve_va(vgpu, page, &pa, &ap))
+		return;
+	svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+	svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap);
+}
+
+/* Does any captured inline write land in this page? */
+static bool svc_gr_page_inline_covers(u64 page)
+{
+	u32 i;
+
+	for (i = 0; i < svc_gr_cand.n_inline; i++) {
+		if ((svc_gr_cand.inline_recs[i].dst_va &
+		     ~(u64)(SEMA_PAGE_SIZE - 1)) == page)
+			return true;
+	}
+	return false;
+}
+
+/*
+ * Copy the current GR capture into the request slot (INFLIGHT is held). Every
+ * launch page is collected and resolved; a page with no resolved PA is only
+ * acceptable when a QMD pointer pair names it and captured inline bytes cover
+ * it (the backend's Tier 2 relocation). Returns 0 or an errno the caller
+ * falls back on.
+ */
+static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
+			   const struct svc_channel *ch)
+{
+	u32 i;
+
+	svc_fwd_req.done = false;
+	svc_fwd_req.hw_rc = 0;
+	memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault));
+	svc_fwd_req.guest = false;
+	svc_fwd_req.gr = true;
+	svc_fwd_req.gr_bridge_done = false;
+	svc_fwd_req.defer = true;
+	svc_fwd_req.relay = false;
+	svc_fwd_req.relay_index = 0;
+	svc_fwd_req.vgpu = vgpu;
+	svc_fwd_req.gr_h_object = ch->h_object;
+	svc_fwd_req.gr_push_va = svc_gr_cand.push_va;
+	svc_fwd_req.gr_push_len = svc_gr_cand.push_len;
+	svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va;
+	svc_fwd_req.gr_rel_flags = svc_gr_cand.rel_flags;
+	svc_fwd_req.gr_rel_va = svc_gr_cand.rel_va;
+	svc_fwd_req.gr_rel_payload = svc_gr_cand.rel_payload;
+	svc_fwd_req.gr_prog_va = svc_gr_cand.prog_va;
+	svc_fwd_req.gr_param_va = svc_gr_cand.param_va;
+	svc_fwd_req.gr_param_size = svc_gr_cand.param_size;
+	memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd,
+	       sizeof(svc_fwd_req.gr_qmd));
+	svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline;
+	if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX)
+		svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX;
+	memcpy(svc_fwd_req.gr_inline, svc_gr_cand.inline_recs,
+	       sizeof(svc_fwd_req.gr_inline));
+	svc_fwd_req.gr_n_pages = 0;
+	svc_fwd_req.gr_page_overflow = false;
+
+	if (svc_gr_cand.param_va)
+		svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32);
+	if (svc_gr_cand.prog_va)
+		svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48);
+	for (i = 0; i < svc_gr_cand.n_pointer; i++)
+		svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i],
+				 svc_gr_cand.pointer_word[i]);
+	for (i = 0; i < svc_gr_cand.n_inline; i++)
+		svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va,
+				 SVC_GR_QMD_WORD_NONE);
+	if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages)
+		return -E2BIG;
+	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+		const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i];
+
+		if (pg->pa)
+			continue;
+		if (pg->qmd_word == SVC_GR_QMD_WORD_NONE)
+			return -ENOENT;
+		if (!svc_gr_page_inline_covers(pg->va))
+			return -ENOENT;
+	}
+	return 0;
+}
+
+/*
+ * Exactly-once CUDA bridge fallback from the request copy (work item or a
+ * failed queue): hand the captured launch to the bridge when enabled. The
+ * latch is per request slot and the slot is single-inflight.
+ */
+static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu)
+{
+	u32 n;
+
+	if (!svc_fwd_req.gr || svc_fwd_req.gr_bridge_done)
+		return;
+	svc_fwd_req.gr_bridge_done = true;
+	if (!gsp_cuda_bridge_enabled())
+		return;
+	n = svc_fwd_req.gr_n_inline;
+	if (n > GSP_BRIDGE_INLINE_MAX)
+		n = GSP_BRIDGE_INLINE_MAX;
+	gsp_cuda_bridge_launch(vgpu, svc_fwd_req.gr_h_object,
+			       svc_fwd_req.gr_qmd, svc_fwd_req.gr_qmd_dev_va,
+			       svc_fwd_req.gr_push_va,
+			       (const struct gsp_bridge_inline *)
+				       svc_fwd_req.gr_inline, n);
+	svc.fwd_gr_bridge++;
+}
+
+/*
+ * GR request path (gsp_phys_gr_forward=1 only), called by svc_fwd_request()
+ * for engine_type==0 channels. Queues one physical submission when the GR
+ * backend is ready; otherwise consumes the capture through the bridge
+ * fallback. Reuses the CE INFLIGHT/paused/hold exclusion so GR and CE
+ * forwarding can never overlap; the caller engages the release hold on a
+ * true return. Service-thread only.
+ */
+static bool svc_gr_request(struct nv_vgpu_device *vgpu,
+			   const struct svc_channel *ch)
+{
+	int rc;
+
+	if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid)
+		return false;
+	svc.fwd_gr_req++;
+	if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) {
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_fallback++;
+		svc_gr_bridge_fallback(vgpu);
+		return false;
+	}
+	/*
+	 * Never reuse the request slot while a hold is active; the held
+	 * request's done/hw_rc are still being read. The capture falls back.
+	 */
+	if (svc_fwd_hold.active) {
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_fallback++;
+		svc_gr_bridge_fallback(vgpu);
+		return false;
+	}
+	mutex_lock(&svc_fwd_lock);
+	if (svc_fwd_paused ||
+	    test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) {
+		mutex_unlock(&svc_fwd_lock);
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_fallback++;
+		svc_gr_bridge_fallback(vgpu);
+		return false;
+	}
+	rc = svc_gr_req_fill(vgpu, ch);
+	if (rc) {
+		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+		mutex_unlock(&svc_fwd_lock);
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_fallback++;
+		if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1)
+			pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n",
+				rc, ch->h_object, svc_gr_cand.push_va);
+		svc_gr_bridge_fallback(vgpu);
+		return false;
+	}
+	/*
+	 * The candidate is consumed: the request copy owns it from here, so a
+	 * later capture/replay cannot re-offer it and the bridge is only
+	 * reached through the request's own latch.
+	 */
+	svc_gr_cand.offered = true;
+	svc_gr_cand.valid = false;
+	svc_fwd_requests++;
+	if (!svc_fwd_wq) {
+		svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0);
+		if (!svc_fwd_wq) {
+			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+			svc_fwd_dropped++;
+			mutex_unlock(&svc_fwd_lock);
+			svc_gr_req_bridge_fallback(vgpu);
+			return false;
+		}
+	}
+	if (!queue_work(svc_fwd_wq, &svc_fwd_work)) {
+		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+		svc_fwd_dropped++;
+		mutex_unlock(&svc_fwd_lock);
+		svc_gr_req_bridge_fallback(vgpu);
+		return false;
+	}
+	mutex_unlock(&svc_fwd_lock);
+	return true;
+}
+
+/*
  * Called only from the service thread (the pump). Returns true when a guest
  * (hardware-copy) request was queued; the caller then decides whether to hold
  * the entry for its deferred releases. All wave-1 gates are unchanged.
@@ -8741,7 +9377,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
 			    const struct svc_channel *ch)
 {
 	if (!ch->engine_type)
-		return false;
+		return svc_gr_request(vgpu, ch);
 	if (!gsp_phys_work_forward_enabled())
 		return false;
 	/* Classify this decode pass's candidate for the run summary. */
@@ -8845,7 +9481,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
 		return false;
 	}
 	mutex_unlock(&svc_fwd_lock);
-	return svc_fwd_req.guest;
+	return svc_fwd_req.guest || svc_fwd_req.gr;
 }
 
 /*
@@ -8998,6 +9634,27 @@ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched,
 }
 
 /*
+ * GR forwarding counters for the gsp-phys-state work_gr line (see
+ * gsp_sema_service.h). Same plain-read discipline as the stats above.
+ */
+void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback,
+				   u64 *refused, u64 *timeout, u64 *fault)
+{
+	if (req)
+		*req = svc.fwd_gr_req;
+	if (ok)
+		*ok = svc.fwd_gr_ok;
+	if (fallback)
+		*fallback = svc.fwd_gr_fallback;
+	if (refused)
+		*refused = svc.fwd_gr_refused;
+	if (timeout)
+		*timeout = svc.fwd_gr_timeout;
+	if (fault)
+		*fault = svc.fwd_gr_fault;
+}
+
+/*
  * P6c: true while this channel owns an active RELAY hold. The pump's
  * per-pass notifier is suppressed for such a channel so the hold's verified
  * success edge (or the next pass after a failure resolution) is the single
@@ -9479,6 +10136,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									  idx,
 									  va,
 									  injected,
+									  entry_replay,
 									  entry_replay ?
 									  NULL :
 									  &entry_done);
@@ -9684,10 +10342,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 
 									if (!svc_rescue_push_page(vgpu, ch, va, len) &&
 									    !svc_read_va_push(vgpu, ch, va, pb, len))
-										svc_decode_stream(vgpu, pb,
-												  len, 0, false,
-												  ch, idx, va, true,
-												  NULL);
+									svc_decode_stream(vgpu, pb,
+											  len, 0, false,
+											  ch, idx, va, true,
+											  true,
+											  NULL);
 									rel_seen =
 										(svc_rel_count != rel_n2);
 									/*
@@ -9713,6 +10372,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 											svc_decode_stream(vgpu, pb,
 													  len, 0, false,
 													  ch, idx, va, true,
+													  true,
 													  NULL);
 										rel_seen =
 											(svc_rel_count != rel_n2);
@@ -9948,6 +10608,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 											  SVC_ENTRY_NONE,
 											  va,
 											  false,
+											  false,
 											  NULL);
 									if (svc_rel_count != rel_n_before)
 										ch->last_rel_va =