vgsp: GR forwarding static-review fixes (GR/CE slot bit, release VA->PA, shadow refusal)

Five defects found in a static review of the default-off GR forwarding path
(no hardware test possible; GPU wedged):

- clear the shared request slot's GR mode bit when a CE request is filled,
  so a prior GR request can never make a later CE push run the GR work
  function with stale fields;
- resolve the QMD release0 guest VA to a guest PA on the service thread
  (counter pin/derive, else the trusted resolver) and only convert PA->HPA
  in the work item; the previous code fed the VA to the PA-based translator
  and would write release0 into an unrelated guest page;
- take the refusal path (goto gr_page_unmapped) on every Tier-2 shadow
  failure instead of continuing to submit a half-built method stream;
- consume the QMD run on replay passes when the GR forward is enabled so
  replayed QMD words are never reinterpreted as CE method headers;
- only queue a captured GR candidate against the channel that captured it.

Build clean (make -C kernel, W=1 only pre-existing warnings).

2 files changed, 74 insertions, 15 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index bf70cd5..ebe873b 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -11295,20 +11295,20 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 
 			if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
 				gsp_phys_gr_pool_exhausted++;
-				continue;
+				goto gr_page_unmapped;
 			}
 			if (!sh->pages) {
 				ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
 							   GSP_PHYS_RMCALL_GR_SHADOW_PAGES);
 				if (ret) {
 					gsp_phys_gr_pool_exhausted++;
-					continue;
+					goto gr_page_unmapped;
 				}
 				gsp_phys_chunk_zero(sh);
 			}
 			sh_page = gsp_phys_chunk_page_va(sh, n_shadow);
 			if (!sh_page)
-				continue;
+				goto gr_page_unmapped;
 			memset(sh_page, 0, GSP_PHYS_PAGE_SIZE);
 			for (j = 0; j < req->n_inl; j++) {
 				const struct gsp_phys_gr_inline *inl =
@@ -11351,6 +11351,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				}
 			}
 		}
+gr_page_unmapped:
 		if (!mapped) {
 			mutex_unlock(&gsp_phys_rm_lock);
 			gsp_phys_gr_refused++;
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 2b7fa80..b43b222 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4717,7 +4717,7 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload)
 static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			  struct svc_push_state *st, u32 addr, u32 count,
 			  const u8 *data, const u8 *push, u32 push_len,
-			  bool trusted)
+			  bool trusted, bool replay)
 {
 	u32 i;
 
@@ -4776,10 +4776,10 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			memcpy(recs[n].data, rec->data, rec->len);
 			n++;
 		}
-		if (gsp_phys_gr_forward_enabled() && trusted)
+		if (gsp_phys_gr_forward_enabled() && trusted && !replay)
 			svc_gr_cand_capture(ch, st, recs, n, push, push_len,
 					    st->push_va);
-		else
+		else if (!replay || !gsp_phys_gr_forward_enabled())
 			gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd,
 					       st->qmd_dev_va, st->push_va,
 					       recs, n);
@@ -4986,7 +4986,6 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		 * must fall through to the CE decoder.
 		 */
 		if (ch->engine_type == 0 && subch >= 1 &&
-		    (!gsp_phys_gr_forward_enabled() || !replay) &&
 		    (gsp_cuda_bridge_enabled() ||
 		     gsp_phys_gr_forward_enabled()) &&
 		    addr >= SVC_QMD_ADDR_A &&
@@ -4994,7 +4993,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		    (st.qmd_have ||
 		     (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) {
 			svc_qmd_accum(vgpu, ch, &st, addr, count,
-				      buf + off + 4, buf, len, trusted);
+				      buf + off + 4, buf, len, trusted, replay);
 			off += 4 + count * 4;
 			continue;
 		}
@@ -7991,6 +7990,14 @@ struct svc_fwd_req {
 	u32 gr_rel_flags;
 	u64 gr_rel_va;
 	u32 gr_rel_payload;
+	/*
+	 * The QMD release0 page, resolved VA->guest PA on the service thread
+	 * (gr_rel_va is a guest VA, never a PA); the work item turns
+	 * gr_rel_pa into an HPA with the aperture below. 0 = unresolved, in
+	 * which case release0 falls back to the module semaphore.
+	 */
+	u64 gr_rel_pa;
+	u8 gr_rel_ap;		/* SVC_APERTURE_* of gr_rel_pa */
 	u64 gr_prog_va;
 	u64 gr_param_va;
 	u32 gr_param_size;
@@ -8840,14 +8847,16 @@ static void svc_fwd_gr_work(void)
 	    (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ONE_WORD) &&
 	    !(svc_fwd_req.gr_rel_flags & SVC_GR_REL_REDUCTION) &&
 	    (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ENABLE)) {
-		u64 page = svc_fwd_req.gr_rel_va &
-			   ~(u64)(SEMA_PAGE_SIZE - 1);
 		u32 roff = (u32)(svc_fwd_req.gr_rel_va &
 				 (SEMA_PAGE_SIZE - 1));
-
-		if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, SVC_APERTURE_SYS, page,
-					0, SEMA_PAGE_SIZE, &rel_hpa, &off) ||
-		    off)
+		u8 rel_kind = (svc_fwd_req.gr_rel_ap == SVC_APERTURE_FB) ?
+			      SVC_APERTURE_FB : SVC_APERTURE_SYS;
+
+		if (svc_fwd_req.gr_rel_pa &&
+		    (svc_fwd_operand_hpa(svc_fwd_req.vgpu, rel_kind,
+					 svc_fwd_req.gr_rel_pa, 0,
+					 SEMA_PAGE_SIZE, &rel_hpa, &off) ||
+		     off))
 			rel_hpa = 0;
 		/*
 		 * Forward-only: a release word already at/ahead of the payload
@@ -9223,6 +9232,40 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 	svc_fwd_req.gr_rel_flags = svc_gr_cand.rel_flags;
 	svc_fwd_req.gr_rel_va = svc_gr_cand.rel_va;
 	svc_fwd_req.gr_rel_payload = svc_gr_cand.rel_payload;
+	/*
+	 * Resolve the release0 VA to a guest PA now, on the service thread
+	 * (the work item only does PA->HPA). A GR completion counter resolves
+	 * through the injected pin / structural derivation exactly like the
+	 * counter machinery; anything else through the trusted resolver. A
+	 * miss leaves gr_rel_pa 0 and release0 falls back to the module
+	 * semaphore (host-observed completion, release still software-applied
+	 * by the hold).
+	 */
+	svc_fwd_req.gr_rel_pa = 0;
+	svc_fwd_req.gr_rel_ap = SVC_APERTURE_SYS;
+	if (svc_gr_cand.rel_va) {
+		u64 pa = 0;
+		u32 ap = SVC_APERTURE_SYS;
+		bool ok = false;
+
+		if (svc_counter_va(svc_gr_cand.rel_va)) {
+			if (!svc_ipin_lookup(svc_gr_cand.rel_va, &pa, &ap) &&
+			    ap == SVC_APERTURE_SYS)
+				ok = true;
+			else if (gsp_sema_counter_derive &&
+				 !svc_counter_derive(vgpu, svc_gr_cand.rel_va,
+						     &pa))
+				ok = true;
+		} else if (!svc_resolve_va(vgpu, svc_gr_cand.rel_va, &pa,
+					   &ap)) {
+			ok = true;
+		}
+		if (ok) {
+			svc_fwd_req.gr_rel_pa =
+				pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+			svc_fwd_req.gr_rel_ap = (u8)svc_ap_of(ap);
+		}
+	}
 	svc_fwd_req.gr_prog_va = svc_gr_cand.prog_va;
 	svc_fwd_req.gr_param_va = svc_gr_cand.param_va;
 	svc_fwd_req.gr_param_size = svc_gr_cand.param_size;
@@ -9299,7 +9342,14 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 {
 	int rc;
 
-	if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid)
+	/*
+	 * The candidate is per-channel: a capture left unconsumed by an
+	 * earlier pass (e.g. the entry was held before its request) must never
+	 * be queued against a different channel's handle by that channel's
+	 * own end-of-pass request call.
+	 */
+	if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid ||
+	    svc_gr_cand.h_object != ch->h_object)
 		return false;
 	svc.fwd_gr_req++;
 	if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) {
@@ -9430,6 +9480,14 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
 	svc_fwd_req.done = false;
 	svc_fwd_req.hw_rc = 0;
 	memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault));
+	/*
+	 * A CE request must clear the GR mode bit: a prior GR request leaves
+	 * gr=true in the shared slot, and svc_fwd_work_fn dispatches on it, so
+	 * without this a CE request would run the stale GR branch (and the
+	 * return value below would report a queue that never happened).
+	 */
+	svc_fwd_req.gr = false;
+	svc_fwd_req.gr_bridge_done = false;
 	svc_fwd_req.guest = svc_fwd_cand.valid &&
 			    gsp_phys_guest_mem_enabled();
 	svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed &&