vgsp: eng64 - chain the release behind the pack carrier and name a stalled carrier's gate

c779: the pack-continuation carrier completed the pack
(MME-CARRY-COMPLETE gen=4 off=2799/2799) and armed the release
(MME-CARRY-ARM mode=release), but its launch poll held the single
forwarding slot for 2801 ms; the armed release then deferred
request-inflight until the beef channel unregistered at +2.2 s, so the
release ended in MME-CARRY-HOLD reason=no-channel.

- svc_fwd_gr_work: after a successful synthetic pack carrier, wait out
  the release grace in the same slot, claim it and run the release work
  directly (chained MME-CARRY-FIRE mode=release ... receipt). Gate:
  sema_gl_mme_release_chain (default 1; release carrier still default 0).
- pump pack-carry block: an armed due carrier that cannot go out now
  names its blocker (MME-CARRY-WAIT reason=no-channel/client-candidate/
  request-inflight/fe-defer), so the c781-class silent stall is visible.
- booter: gsp_phys_gr_mme_release_waiting()/chain_enabled() helpers.

c783-c785 (a03a68ea build) were all shallow (author=0, no MME window).

3 files changed, 96 insertions, 3 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 4cf1c55..74b0300 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -19011,6 +19011,23 @@ MODULE_PARM_DESC(sema_gl_mme_release_sa_arm,
 	"Re-emit the START-ADDRESS RAM pairs (incl. golden slot 34 -> IR 0x265) before a saved-window release launch so CALL_MME_MACRO(34) resolves to DRAW_IAB (default 0; needs sema_gl_mme_pack_release=1)");
 
 /*
+ * Release chaining (sema_gl_mme_release_chain): the host pack-continuation
+ * carrier's launch poll owns the single forwarding slot for up to
+ * sema_mme_fast_ms (c779 elapsed 2801 ms), and the client's 3D channel can
+ * unregister before the next service pass fires the armed release (c779:
+ * release ARM at +0.1 s, beef channel unregistered ~2.2 s later ->
+ * MME-CARRY-HOLD reason=no-channel). With this on, a successful synthetic
+ * pack carrier claims the armed release after its settle grace and runs the
+ * release work in the same slot, while the channel is still alive. Default 1;
+ * the release carrier itself is default off.
+ */
+static bool gsp_phys_rmcall_gr_mme_release_chain = true;
+module_param_named(sema_gl_mme_release_chain,
+		   gsp_phys_rmcall_gr_mme_release_chain, bool, 0644);
+MODULE_PARM_DESC(sema_gl_mme_release_chain,
+	"Run the armed saved-window release in the same forwarding slot right after a successful host pack-continuation carrier (default 1; needs sema_gl_mme_pack_release=1)");
+
+/*
  * DRAW_IAB preference (release carrier follow-up): the held-call window that
  * matters is the DRAW window, whose method stream carries the
  * CALL_MME_MACRO(34) run that expands to DRAW_IAB (mac=0x22).
@@ -19470,6 +19487,24 @@ bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win)
 	return true;
 }
 
+/* An armed release is still owed (before its grace deadline too). */
+bool gsp_phys_gr_mme_release_waiting(void)
+{
+	struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release;
+	unsigned long flags;
+	bool waiting;
+
+	spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags);
+	waiting = s->valid && s->armed && !s->fired && !s->consumed;
+	spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags);
+	return waiting;
+}
+
+bool gsp_phys_gr_mme_release_chain_enabled(void)
+{
+	return READ_ONCE(gsp_phys_rmcall_gr_mme_release_chain);
+}
+
 /* The release is still claimed and not cancelled (builder safety gate). */
 static bool gsp_phys_gr_mme_release_active(void)
 {
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 21eb1fc..30aef8f 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -596,6 +596,8 @@ void gsp_phys_gr_mme_carry_failed(int rc);
  * through the normal fill/build path with the CALL runs released.
  */
 bool gsp_phys_gr_mme_release_peek(u32 *h_object);
+bool gsp_phys_gr_mme_release_waiting(void);
+bool gsp_phys_gr_mme_release_chain_enabled(void);
 bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win);
 void gsp_phys_gr_mme_release_rearm(void);
 bool gsp_phys_gr_mme_release_is_arming(u32 h, u64 push);
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 603e9f0..435386e 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -39161,6 +39161,46 @@ static void svc_fwd_gr_work(void)
 		}
 	}
 
+	/*
+	 * Chain the saved-window release behind a successful host pack
+	 * carrier: the carrier's launch poll owns the single forwarding slot
+	 * for up to sema_mme_fast_ms, and the client's 3D channel may tear
+	 * down before the next service pass fires the armed release (c779:
+	 * release ARM at +0.1 s, beef channel unregistered ~2.2 s later ->
+	 * MME-CARRY-HOLD reason=no-channel). Wait out the release's settle
+	 * grace inside this slot, claim it, and run the release work
+	 * directly, so the saved window reaches the engine while the channel
+	 * is still alive. The release work owns the outcome publish and the
+	 * INFLIGHT clear.
+	 */
+	if (!hw_rc && svc_fwd_req.gr_synthetic && !svc_fwd_req.gr_release &&
+	    gsp_phys_gr_mme_release_chain_enabled()) {
+		unsigned long deadline = jiffies + msecs_to_jiffies(500);
+		u32 rheld = 0, rwin = 0;
+		bool took = false;
+
+		while (gsp_phys_gr_mme_release_waiting() &&
+		       time_before(jiffies, deadline)) {
+			if (gsp_phys_gr_mme_release_take(
+				    svc_fwd_req.gr_h_object, &rheld, &rwin)) {
+				took = true;
+				break;
+			}
+			msleep(10);
+		}
+		if (took) {
+			pr_info("vgsp-gr: MME-CARRY-FIRE mode=release held=%u win=%u h=0x%x push=0x%llx (chained after the pack carrier)\n",
+				rheld, rwin, svc_fwd_req.gr_h_object,
+				(unsigned long long)SVC_GR_CARRIER_PUSH_VA);
+			svc_fwd_req.gr_release = true;
+			svc_fwd_req.gr_synthetic = true;
+			svc_fwd_req.done = false;
+			svc_fwd_req.hw_rc = 0;
+			svc_fwd_gr_release_work();
+			return;
+		}
+	}
+
 	svc_fwd_req.fault = cp.fault;
 	svc_fwd_req.hw_rc = hw_rc;
 	/*
@@ -50113,6 +50153,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		u32 carry_h = 0;
 
 		if (gsp_phys_gr_mme_carry_peek(&carry_h)) {
+			const char *why = "no-channel";
 			int ci;
 
 			for (ci = 0; ci < n; ci++) {
@@ -50120,16 +50161,31 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 
 				if (c->h_object != carry_h || c->engine_type)
 					continue;
-				if (svc_gr_cand_unoffered_for(carry_h))
+				why = NULL;
+				if (svc_gr_cand_unoffered_for(carry_h)) {
+					why = "client-candidate";
 					break;
+				}
 				if (svc_fwd_hold.active ||
-				    test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state))
+				    test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) {
+					why = "request-inflight";
 					break;
-				if (gsp_phys_gr_mme_fe_defer_wanted(carry_h, 2))
+				}
+				if (gsp_phys_gr_mme_fe_defer_wanted(carry_h, 2)) {
+					why = "fe-defer";
 					break;
+				}
 				(void)svc_gr_carrier_request(vgpu, c, false);
 				break;
 			}
+			/*
+			 * An armed carrier that is due but cannot go out names
+			 * its blocker (c781: it sat armed until the beef
+			 * channel unregistered, with no receipt at all).
+			 */
+			if (why)
+				pr_info_ratelimited("vgsp-gr: MME-CARRY-WAIT reason=%s h=0x%x\n",
+						    why, carry_h);
 		}
 	}