vgsp: m178 GR launch reliability + diagnostics; tools: pin latching robustness

Kernel: authorship instrumentation + swrel gate for in-flight GR release VAs;
first-poll shortening + fast extended re-arm (params gr_retry_fast=1,
gr_first_poll_ms=2500); per-attempt win0 bank recompute + shadow recopy in the
outcome-retry loop; poll-break/deposit-readback diagnostics; C1-C10 sema
diagnostics (crel-entry, relx, opdef-clear, walk, pub-adv, hold-fallback,
busy-defer, CAND-OVERWRITE, inline recs); busy-defer hold instead of
software-release consume; counter fast path; QMD word 48/49 skip.
Tools: heartbeat thread, scanner-alive probe on reinstall, full periodic
re-assert, #nopid handling, per-client reset fix (debounce), dead-pid drop,
supervisor grace+threshold.

1 files changed, 311 insertions, 14 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 7b12ecc..01f94cf 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -805,6 +805,7 @@ struct sema_svc {
 						 * deliverable exact slot */
 	u64			counter_rel_applied;	/* releases to registered counters */
 	u64			counter_rel_dbg;	/* rel-apply diagnostic budget */
+	u64			crel_entry_dbg;	/* C1: rel-apply entry budget, 512/session */
 	u64			counter_noslot_dbg;	/* "no slot registered" budget,
 						 * separate so it cannot hide the
 						 * written/skip lines */
@@ -831,6 +832,8 @@ struct sema_svc {
 	u64			vidmem_writes;		/* semaphores written in vidmem */
 	u64			gpfifo_log;		/* channel-state log budget */
 	u64			gpfifo_entry_log;	/* per-entry log budget */
+	u64			walk_dbg;	/* C4: GR put/walk witness budget */
+	u64			gr_rel_auth_log;	/* F1: GR-REL-AUTHOR budget, 256/session */
 	u64			doorbell_count;		/* NV_VIRTUAL_FUNCTION_DOORBELL pokes */
 	u32			doorbell_token;		/* last token written */
 	u32			doorbell_log;		/* token log budget */
@@ -1380,6 +1383,17 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu,
 static void svc_opdef_reset_all(void);
 static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx);
 static u32 svc_opdef_log;	/* m176: forward decl for the session flush */
+/*
+ * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10):
+ * small, reset per session where they are service-thread state, so a later
+ * client's strand is never blacked out by the first client's spend.
+ */
+static u32 svc_relx_log;	/* C2: deferred-launch apply entry witness */
+static u32 svc_opdef_clr_log;	/* C3: opdef-clear caller witness */
+static u32 svc_busy_defer_log;	/* C8: GR busy-defer timeout witness */
+static u32 svc_hold_fb_log;	/* C7: hold failure/timeout fallback witness */
+static u32 svc_cand_ovw_log;	/* C9: candidate overwrite witness */
+static u32 svc_inline_recs_log;	/* C10: inline-recs digest witness */
 
 void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object)
 {
@@ -3415,6 +3429,13 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
 
 static void svc_pending_add(u64 va, u32 payload, u32 op);
 
+/*
+ * F1: GR release-word authorship checks. Defined next to the svc_fwd_req
+ * slot; forward-declared because the sweep writer runs earlier in the file.
+ */
+static bool svc_gr_rel_match(u64 va, u32 *payload);
+static bool svc_gr_rel_inflight(u64 va, u32 *payload);
+
 static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val)
 {
 	u64 addr = 0, old_addr;
@@ -4528,6 +4549,17 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
 {
 	u32 i, n;
 
+	/*
+	 * C9 (GAP-2 A4): a capture over a still-valid candidate (never
+	 * offered) destroys an unforwarded launch - the single-slot
+	 * overwrite hazard. Bounded, unconditional (gsp_sema_dbg is defined
+	 * later in the file and this site may not reference it).
+	 */
+	if (svc_gr_cand.valid && svc_cand_ovw_log < 64) {
+		svc_cand_ovw_log++;
+		pr_info("sema-gr: CAND-OVERWRITE old_h=0x%x old_push=0x%llx new_push=0x%llx\n",
+			svc_gr_cand.h_object, svc_gr_cand.push_va, push_va);
+	}
 	memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
 	svc_gr_cand.valid = true;
 	svc_gr_cand.h_object = ch->h_object;
@@ -5668,6 +5700,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 	bool skip;
 	int i;
 	u32 dbg = 0;
+	u64 rel_pa = 0;
+	u64 cnt_pa = 0;
 
 	if (!h || !payload) {
 		/*
@@ -5705,12 +5739,23 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 			 * FORENSICS-grv7-counter F1). Fall back to counter_pa
 			 * when the release slot could not be resolved.
 			 */
-			pa = svc.chan[i].release_pa ?
-				svc.chan[i].release_pa : svc.chan[i].counter_pa;
+			rel_pa = svc.chan[i].release_pa;
+			cnt_pa = svc.chan[i].counter_pa;
+			pa = rel_pa ? rel_pa : cnt_pa;
 			break;
 		}
 	}
 	spin_unlock_irqrestore(&svc.chan_lock, flags);
+	/*
+	 * C1 (GAP-1 H3/H4): was the apply entered, and with which slot?
+	 * Budget 512/session (reset in svc_session_flush); the per-branch
+	 * prints below stay on their own counter_rel_dbg budget.
+	 */
+	if (gsp_sema_dbg && svc.crel_entry_dbg < 512) {
+		svc.crel_entry_dbg++;
+		pr_info("sema-diag: crel-entry h=0x%x va=0x%llx pay=%u inc=%d rel_pa=0x%llx cnt_pa=0x%llx\n",
+			h, sem_va, payload, inc ? 1 : 0, rel_pa, cnt_pa);
+	}
 	if (!pa) {
 		/*
 		 * Pool (UVM tracking) semaphores have no counter slot. The
@@ -6314,6 +6359,22 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
 				 u32 payload, u32 cur, bool pending,
 				 bool inc, bool inc_apply)
 {
+	/*
+	 * P4 (GAP-1 H3): the exact-slot counter delivery runs before every
+	 * path that can early-return, so a GR completion-counter release can
+	 * never be skipped by the generic write's outcome. The generic
+	 * write/learn tail below is skipped entirely for counter VAs: an
+	 * increment release is not idempotent, so it must apply exactly once
+	 * (the early return here is that exactly-once edge).
+	 */
+	if (ch->engine_type == 0 && svc_counter_va(sem_va)) {
+		if (pending || inc_apply)
+			svc.pushes_pending++;
+		svc_counter_learn(vgpu, ch->h_object, sem_va);
+		svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
+					  inc && inc_apply);
+		return false;
+	}
 	if (pending || inc_apply) {
 		svc.pushes_pending++;
 		if (inc)
@@ -6325,6 +6386,8 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
 		 * generic translation can be a stale content-scanned page;
 		 * those releases go through the injected-pin-backed exact slot
 		 * (svc_counter_learn / svc_counter_release_apply) below.
+		 * Counter VAs are handled by the early block above; the check
+		 * stays as belt-and-braces for the generic write.
 		 */
 		if (ch->engine_type == 0 && svc_counter_va(sem_va)) {
 			/* handled by the exact-slot path */
@@ -6524,6 +6587,24 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			memcpy(recs[n].data, rec->data, rec->len);
 			n++;
 		}
+		/*
+		 * C10 (GAP-2 A5): duplicate destinations in the newest-first
+		 * inline capture decide which record wins the backend's
+		 * overlay; bounded and unconditional (gsp_sema_dbg is defined
+		 * later in the file and this site may not reference it).
+		 */
+		if (svc_inline_recs_log < 64) {
+			u32 dup = 0, a2, b2;
+
+			for (a2 = 0; a2 < n; a2++)
+				for (b2 = a2 + 1; b2 < n; b2++)
+					if (recs[a2].dst_va ==
+					    recs[b2].dst_va)
+						dup++;
+			svc_inline_recs_log++;
+			pr_info("sema-gr: inline recs n=%u dup_dst=%u\n",
+				n, dup);
+		}
 		if (gsp_phys_gr_forward_enabled() && trusted && !replay)
 			svc_gr_cand_capture(ch, st, recs, n, push, push_len,
 					    st->push_va);
@@ -7978,6 +8059,9 @@ static void svc_session_flush(void)
 	 * because the counters were already exhausted by the first client).
 	 */
 	svc.counter_rel_dbg = 0;
+	svc.crel_entry_dbg = 0;
+	svc.walk_dbg = 0;
+	svc.gr_rel_auth_log = 0;
 	svc.counter_noslot_dbg = 0;
 	svc.sem256_log = 0;
 	svc.xfer_raw_log = 0;
@@ -8051,6 +8135,13 @@ static void svc_session_flush(void)
 	/* m176: re-arm the opdef diagnostic budget (it was never per-session,
 	 * so a late client's strand could be silent). */
 	svc_opdef_log = 0;
+	/* One-cycle diag budgets (C2/C3/C7/C8/C9/C10) are per-session too. */
+	svc_relx_log = 0;
+	svc_opdef_clr_log = 0;
+	svc_busy_defer_log = 0;
+	svc_hold_fb_log = 0;
+	svc_cand_ovw_log = 0;
+	svc_inline_recs_log = 0;
 	/* m176: new session generation - channels and scrubbers re-stamp on
 	 * registration and never inherit per-session learned state. */
 	svc_sess_gen++;
@@ -8322,6 +8413,25 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *
 		}
 
 		/*
+		 * F1-b: while the forwarded GR launch this release belongs to
+		 * is still in flight, the sweep must not author its QMD
+		 * completion word - the backend poll would break on a
+		 * host-authored ack (r198: qmd_rel satisfied ~6 ms after the
+		 * doorbell, zero engine writes). Defer: nothing is written or
+		 * consumed here; the release stays in the guest page and later
+		 * sweep passes re-see it, while the hold/fallback path owns a
+		 * failed hardware copy.
+		 */
+		if (gsp_sema_gr_swrel_gate && svc_gr_rel_inflight(va, NULL)) {
+			if (svc.gr_rel_auth_log < 256) {
+				svc.gr_rel_auth_log++;
+				pr_info("sema-diag: GR-REL-AUTHOR src=sweep va=0x%llx pay=%u h=0x%x DEFER\n",
+					va, payload, 0u);
+			}
+			continue;
+		}
+
+		/*
 		 * A release at page offset 0x10 with payload 1 in front of a
 		 * channel's first push identifies the page as that channel's
 		 * push-buffer base (see svc_pb_add).
@@ -8330,6 +8440,18 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *
 		    (lo & 0xfff) < 0x20 && svc_pb_first_push(buf))
 			svc_pb_add(pa_page, lo & 0xfff);
 
+		/*
+		 * F1-a: tag software authorship of the forwarded launch's
+		 * release word (gate off, or the request already resolved and
+		 * the forward progress rule still allows the write).
+		 */
+		if (svc_gr_rel_match(va, NULL) &&
+		    svc.gr_rel_auth_log < 256) {
+			svc.gr_rel_auth_log++;
+			pr_info("sema-diag: GR-REL-AUTHOR src=sweep va=0x%llx pay=%u h=0x%x\n",
+				va, payload, 0u);
+		}
+
 		if (svc_sem_write(vgpu, 0, va, payload) == 0) {
 			svc.releases_applied++;
 			svc.post_hint_until = jiffies + 10 * HZ;
@@ -10066,6 +10188,18 @@ bool gsp_sema_counter_derive = true;
 unsigned gsp_sema_gr_grace_ms = 8000;
 
 /*
+ * F1-b: GR release-word authorship gate (default 1). While a forwarded GR
+ * launch is in flight, the RAM-content sweep must not author the very QMD
+ * completion release the backend poll is waiting for: a sweep write could
+ * satisfy the poll before the engine retires the QMD (false completion,
+ * silent corruption). With the gate on, such a write is deferred - the
+ * release stays in RAM and later sweep passes re-see it; when the hardware
+ * copy resolves, the hold/fallback path applies it. 0 restores the
+ * unconditional sweep apply for an A/B.
+ */
+bool gsp_sema_gr_swrel_gate = true;
+
+/*
  * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode
  * saw QMD words but built no launch candidate for its channel (a replay-
  * suppressed capture, a partial/split QMD, or a push page the pin stream has
@@ -11363,6 +11497,15 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
 			ch->counter_last = cur;
 		return;
 	}
+	/*
+	 * C5 (GAP-1 H4 secondary): the put-based publish is about to advance
+	 * the same word the decoded releases target; a pre-advance here can
+	 * make a later absolute release a silent skip. Witness every advance.
+	 */
+	if (gsp_sema_dbg)
+		pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n",
+			ch->h_object, ch->counter_pa, val, cur,
+			ch->counter_last);
 	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS,
 			 &val, sizeof(val)))
 		return;
@@ -11716,6 +11859,34 @@ static DEFINE_MUTEX(svc_fwd_lock);
 static bool svc_fwd_paused;
 
 /*
+ * F1 helpers: does a software release write target the release word of the
+ * forwarded GR launch the request slot holds? The slot is written only by the
+ * service thread (this code also runs there) and is never reused while the
+ * hold is active, so no lock is needed - same convention as the existing
+ * svc_fwd_hold_gr_release()/svc_fwd_hold_expired() reads. svc_gr_rel_match()
+ * covers the just-resolved hold (done may already be true); the inflight
+ * variant additionally requires the SVC_FWD_INFLIGHT bit and an unresolved
+ * work item, i.e. the hardware copy the poll is waiting on is still running.
+ */
+static bool svc_gr_rel_match(u64 va, u32 *payload)
+{
+	if (!va || !svc_fwd_req.gr || svc_fwd_req.gr_rel_va != va)
+		return false;
+	if (payload)
+		*payload = svc_fwd_req.gr_rel_payload;
+	return true;
+}
+
+static bool svc_gr_rel_inflight(u64 va, u32 *payload)
+{
+	if (!svc_gr_rel_match(va, payload))
+		return false;
+	if (!test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state))
+		return false;
+	return !smp_load_acquire(&svc_fwd_req.done);
+}
+
+/*
  * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve
  * the page-aligned GPA through the guest-RAM backing file; VID operands
  * resolve the shadow byte offset through the vmalloc backing of BAR1 (the
@@ -11965,8 +12136,19 @@ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu,
 	bool pending = r->op != 0;
 	bool inc = DMA_SEM_REDUCTION_EN(r->dma) &&
 		   DMA_SEM_REDUCTION(r->dma) == SEM_REDUCTION_INC;
+	int rd = svc_sem_read(vgpu, r->va, &cur);
 
-	if (svc_sem_read(vgpu, r->va, &cur)) {
+	/*
+	 * C2 (GAP-1 H2/H3): prove the opdef apply handed the record to the
+	 * helper and what the software read of the release word returned.
+	 */
+	if (gsp_sema_dbg && svc_relx_log < 128) {
+		svc_relx_log++;
+		pr_info("sema-diag: relx kind=%u op=%u inc_claim=%u va=0x%llx pay=%u rd=%d\n",
+			(unsigned)r->kind, (unsigned)r->op,
+			(unsigned)r->inc_claimed, r->va, r->payload, rd);
+	}
+	if (rd) {
 		svc_pending_add(r->va, r->payload, inc ? 1 : 0);
 		svc_rel_apply_launch(vgpu, ch, r->va, r->payload, 0, pending,
 				     inc, r->inc_claimed);
@@ -12103,8 +12285,31 @@ static struct svc_opdef_ent *svc_opdef_alloc(struct nv_vgpu_device *vgpu,
 	return NULL;
 }
 
-static void svc_opdef_clear(struct svc_opdef_ent *e)
+/* C3 caller tags for the opdef-clear witness (H2: which path cleared). */
+#define SVC_OPDEF_CLR_APPLY	0u
+#define SVC_OPDEF_CLR_FINISH	1u
+#define SVC_OPDEF_CLR_FLUSH	2u
+#define SVC_OPDEF_CLR_DROP	3u
+#define SVC_OPDEF_CLR_STALE	4u
+#define SVC_OPDEF_CLR_COUNTED	5u
+#define SVC_OPDEF_CLR_SWEEP	6u
+#define SVC_OPDEF_CLR_STOP	7u
+
+static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller)
 {
+	/*
+	 * C3 (GAP-1 H2): every clear is witnessed with its caller and the
+	 * slot contents, so a "silent" clear (n=0, or a kind the apply
+	 * switch ignores) is distinguishable from a delivery.
+	 */
+	if (gsp_sema_dbg && e->active && svc_opdef_clr_log < 128) {
+		svc_opdef_clr_log++;
+		pr_info("sema-diag: opdef-clear caller=%u h=0x%x entry=%u n=%u kind0=%u va0=0x%llx pay0=%u\n",
+			caller, e->h_object, e->entry_idx, (unsigned)e->n,
+			e->n ? (unsigned)e->rel[0].kind : 0u,
+			e->n ? e->rel[0].va : 0ull,
+			e->n ? e->rel[0].payload : 0u);
+	}
 	memset(e, 0, sizeof(*e));
 }
 
@@ -12142,7 +12347,7 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu,
 				ch->h_object, entry_idx, e->n);
 		}
 	}
-	svc_opdef_clear(e);
+	svc_opdef_clear(e, SVC_OPDEF_CLR_APPLY);
 }
 
 /*
@@ -12174,7 +12379,7 @@ static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx)
 					ch->h_object, idx, e->n);
 			}
 		}
-		svc_opdef_clear(e);
+		svc_opdef_clear(e, SVC_OPDEF_CLR_FINISH);
 		return;
 	}
 }
@@ -12286,7 +12491,7 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			ch->h_object, entry_idx, e->n);
 	}
 	svc.opdef_expired++;
-	svc_opdef_clear(e);
+	svc_opdef_clear(e, SVC_OPDEF_CLR_FLUSH);
 }
 
 /*
@@ -12338,7 +12543,7 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 					    (u32)(e->n - delivered), e->n);
 		}
 	}
-	svc_opdef_clear(e);
+	svc_opdef_clear(e, SVC_OPDEF_CLR_DROP);
 }
 
 /*
@@ -12359,7 +12564,7 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu,
 			continue;
 		if (e->n)
 			svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n);
-		svc_opdef_clear(e);
+		svc_opdef_clear(e, SVC_OPDEF_CLR_STALE);
 	}
 }
 
@@ -12373,7 +12578,7 @@ static void svc_opdef_drop_counted(struct svc_opdef_ent *e)
 		pr_info_ratelimited("sema-opdef: LOSS entry h=0x%x idx=%u dropped with %u undelivered release(s)\n",
 				    e->h_object, e->entry_idx, e->n);
 	}
-	svc_opdef_clear(e);
+	svc_opdef_clear(e, SVC_OPDEF_CLR_COUNTED);
 }
 
 static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu,
@@ -12459,7 +12664,7 @@ static void svc_opdef_expire_sweep(void)
 			pr_info_ratelimited("sema-opdef: SWEEP-LOSS h=0x%x entry=%u n=%u owner channel gone (releases dropped)\n",
 					    e->h_object, e->entry_idx, e->n);
 		}
-		svc_opdef_clear(e);
+		svc_opdef_clear(e, SVC_OPDEF_CLR_SWEEP);
 	}
 }
 
@@ -12500,6 +12705,16 @@ static void svc_gr_sw_release(struct nv_vgpu_device *vgpu, u32 h, u64 va,
 		return;
 	if (!svc_sem_read(vgpu, va, &cur) && cur >= payload)
 		return;
+	/*
+	 * F1-a: this is the hold/fallback authorship of the forwarded launch's
+	 * release word (src="hold"), budgeted with the sweep tag so one
+	 * 256-line session window covers both authors.
+	 */
+	if (svc_gr_rel_match(va, NULL) && svc.gr_rel_auth_log < 256) {
+		svc.gr_rel_auth_log++;
+		pr_info("sema-diag: GR-REL-AUTHOR src=hold va=0x%llx pay=%u h=0x%x\n",
+			va, payload, h);
+	}
 	if (!svc_sem_write(vgpu, h, va, payload)) {
 		svc.fwd_gr_sw_rel++;
 		pr_info("vgsp-work: GR launch release software-applied h=0x%x va=0x%llx pay=0x%x (no successful hardware copy; forward-only)\n",
@@ -12849,6 +13064,22 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
 		 * done && hw_rc == -ETIMEDOUT; abort/cancel paths never write
 		 * guest memory).
 		 */
+		/*
+		 * C7 (GAP-2 A1): the false-completion edge - a hold whose
+		 * hardware copy failed/timed out still software-applies the
+		 * release set here.
+		 */
+		if (gsp_sema_dbg && svc_hold_fb_log < 64) {
+			svc_hold_fb_log++;
+			pr_info("sema-diag: hold-fallback h=0x%x gr=%d done=%d hw_rc=%d nrel=%u rel0=0x%llx/%u -> sw apply\n",
+				ch->h_object, ch->engine_type == 0 ? 1 : 0,
+				done ? 1 : 0, svc_fwd_req.hw_rc,
+				(unsigned)svc_fwd_hold.nrel,
+				svc_fwd_hold.nrel ?
+					svc_fwd_hold.rel[0].va : 0ull,
+				svc_fwd_hold.nrel ?
+					svc_fwd_hold.rel[0].payload : 0u);
+		}
 		if (done && svc_fwd_req.hw_rc == -ETIMEDOUT &&
 		    svc_fwd_hold.relay_armed)
 			svc_fwd_fault_deliver(hv, ch);
@@ -13846,6 +14077,8 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 		 * in every launch and starved the real operands under the old
 		 * window. Skip them as data candidates; they are
 		 * shadowed/rewritten through gr_pages/Tier 2, not this path.
+		 * P6: the QMD program pair (word 48/49) is launch-internal in
+		 * exactly the same way and is skipped with them.
 		 */
 		if (svc_fwd_req.gr_param_va &&
 		    (base & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
@@ -13859,7 +14092,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 					&svc_fwd_req.gr_pages[i];
 
 				if (pp->qmd_word >= 32u &&
-				    pp->qmd_word <= 47u &&
+				    pp->qmd_word <= 49u &&
 				    (base & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
 				    (pp->va & ~(u64)(SEMA_PAGE_SIZE - 1))) {
 					launch_internal = true;
@@ -14846,6 +15079,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		int rrc;
 		u32 put = 0, get = 0, guard = 0;
 		u32 get_before;
+		u32 put_prev_wit;
 		u64 applied_before;
 		bool held = false;
 
@@ -15066,6 +15300,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			}
 			bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS);
 		}
+		put_prev_wit = ch->put_prev;
 		ch->put_prev = put;
 
 		/*
@@ -15103,6 +15338,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		get_before = ch->last_get;
 		applied_before = svc.releases_applied;
 
+		/*
+		 * C4 (GAP-1 H1): the GR walk witness - does put advance, and
+		 * does last_get reach it? Budget 512/session; fires on a put
+		 * change or while last_get is in the stalled 80..100 band.
+		 */
+		if (gsp_sema_dbg && svc.walk_dbg < 512 &&
+		    (put != put_prev_wit ||
+		     ((u32)ch->last_get >= 80 && (u32)ch->last_get <= 100))) {
+			svc.walk_dbg++;
+			pr_info("sema-diag: walk h=0x%x put=%u get=%u last=%u prev=%u\n",
+				ch->h_object, put, get, (u32)ch->last_get,
+				put_prev_wit);
+		}
+
 		while (ch->last_get != put && guard++ < ch->gpfifo_entries) {
 			u8 entry[SVC_GPFIFO_ENTRY_SIZE];
 			u64 va, idx = ch->last_get;
@@ -15693,8 +15942,56 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 										fwd_guest_queued =
 											svc_fwd_request(vgpu,
 													 ch);
-									if (fwd_guest_queued)
+									if (fwd_guest_queued) {
 										svc.fwd_gr_defer++;
+									} else if (ch->gr_hold_idx != (u32)idx ||
+										   time_before(jiffies,
+											       ch->gr_hold_until)) {
+										u32 hold_ms = gsp_sema_gr_grace_ms ?
+											gsp_sema_gr_grace_ms : 8000;
+
+										/*
+										 * P2 (GAP-2 A3): the wait
+										 * expired without queueing.
+										 * Do NOT fall through to the
+										 * software-release consume -
+										 * the launch was never
+										 * submitted, so authoring
+										 * its release would be a
+										 * false completion. Hold
+										 * exactly like the
+										 * data-stalled path
+										 * (m150/m156); the one-shot
+										 * window lets the historical
+										 * consume resume after it
+										 * expires.
+										 */
+										ch->gr_rehold_idx = (u32)idx;
+										ch->gr_hold_idx = (u32)idx;
+										ch->gr_hold_until = jiffies +
+											msecs_to_jiffies(hold_ms);
+										svc.fwd_gr_defer++;
+										if (svc_busy_defer_log < 64) {
+											svc_busy_defer_log++;
+											pr_info("vgsp-work: GR busy-defer timeout h=0x%x idx=%llu -> hold %u ms\n",
+												ch->h_object,
+												(unsigned long long)idx,
+												hold_ms);
+										}
+										kfree(pb);
+										break;
+									} else if (svc_busy_defer_log < 64) {
+										/*
+										 * C8 (GAP-2 A3): the bounded
+										 * hold window is over; the
+										 * historical consume with the
+										 * software release resumes.
+										 */
+										svc_busy_defer_log++;
+										pr_info("vgsp-work: GR busy-defer timeout h=0x%x idx=%llu -> consume w/ sw release\n",
+											ch->h_object,
+											(unsigned long long)idx);
+									}
 								}
 								/*
 								 * m150: the GR fill held this
@@ -17832,7 +18129,7 @@ void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu)
 	/* Held-operand release slots of this guest cannot be delivered. */
 	for (i = 0; i < SVC_OPDEF_SLOTS; i++)
 		if (svc_opdef[i].active && svc_opdef[i].vgpu == vgpu)
-			svc_opdef_clear(&svc_opdef[i]);
+			svc_opdef_clear(&svc_opdef[i], SVC_OPDEF_CLR_STOP);
 
 	/* Free the per-guest session and scrubber records. */
 	spin_lock_irqsave(&svc_guest_lock, flags);