vgsp: eng59 - saved-window release carrier for the held MME CALL runs (default off)

When a 324 B draw window holds its CALL runs while the 2799-word pack is
incomplete, the pack can complete later (X lane or a 728 B window) but the
next client launch that would release the held calls is a per-frame lottery
(c756: MME-CALL-HOLD 18, pack complete, no next launch -> no REARM/mac=0x22).

Save the held window at MME-CALL-HOLD (raw pre-relocation stream + the
launch capture fields) into module memory. When the pack is complete and
resident and no client launch consumes the release within
sema_gl_mme_pack_release_ms, re-submit the saved capture through the normal
fill/build path with the held CALL runs KEPT (mme_hold_calls=false), no
prelude emission, and gr_synthetic/no guest release. The released
CALL(34) runs DRAW_IAB against the resident pack, the shape c716 proved.

New: sema_gl_mme_pack_release (bool, 0) / _ms (uint, 100); MME-CARRY-SAVE /
ARM/FIRE/COMPLETE/CANCEL/ABORT mode=release receipts. Beef-only, one fire per
pack completion, fail-closed on context/handle changes or launch errors. The
existing pack-continuation carrier is unchanged with its knob off.

Build rc 0; validation pending.

1 files changed, 167 insertions, 28 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 6cb7034..6d72a62 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -36034,6 +36034,14 @@ struct svc_fwd_req {
 	 * path rewinds the pack so the next client window re-emits from word 0.
 	 */
 	bool gr_synthetic;
+	/*
+	 * Saved-window release carrier (sema_gl_mme_pack_release): a synthetic
+	 * request that carries no capture of its own; the work item fills the
+	 * submit request from the booter's saved record and submits it with
+	 * the held CALL runs released. gr_synthetic is also set so the
+	 * outcome/software-release paths never touch it.
+	 */
+	bool gr_release;
 	bool gr_bridge_done;
 	bool gr_page_overflow;
 	/*
@@ -38674,6 +38682,69 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
 
 static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu);
 
+#define SVC_GR_CARRIER_PUSH_VA	0x6d6d6500ULL	/* unique synthetic id */
+#define SVC_GR_CARRIER_STUB_WAIT	0x20010044u	/* INC 0x110 WAIT_FOR_IDLE */
+
+/*
+ * Saved-window release carrier work item (sema_gl_mme_pack_release).
+ *
+ * The booter saved the held-call window + its capture at the MME-CALL-HOLD
+ * point; gsp_phys_gr_mme_release_fill() copies that snapshot into the submit
+ * request and the normal backend fill/build path replays it with the CALL
+ * runs released. No guest release or QMD word is authored (gr_synthetic
+ * suppresses the stall-completion path); the booter's release_complete /
+ * release_failed own the receipts and the one-fire budget.
+ */
+static void svc_fwd_gr_release_work(void)
+{
+	struct gsp_phys_gr_submit gs;
+	struct gsp_phys_backend_copy_result cp = { 0 };
+	struct task_struct *task;
+	int hw_rc;
+
+	memset(&gs, 0, sizeof(gs));
+	if (!gsp_phys_gr_mme_release_fill(&gs)) {
+		/*
+		 * The record was cancelled after the claim (context wipe,
+		 * re-register, gen rewind): never submit a stale snapshot.
+		 */
+		gsp_phys_gr_mme_release_cancel("save-lost");
+		svc_fwd_req.hw_rc = -EINVAL;
+		svc_fwd_req.fault = cp.fault;
+		smp_store_release(&svc_fwd_req.done, true);
+		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+		task = READ_ONCE(svc.task);
+		if (task)
+			wake_up_process(task);
+		return;
+	}
+	/*
+	 * The launch receipts key on the synthetic carrier id; the saved push
+	 * is only the record's provenance.
+	 */
+	gs.push_va = SVC_GR_CARRIER_PUSH_VA;
+	hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp);
+	if (hw_rc) {
+		svc.fwd_gr_fallback++;
+		if (hw_rc == -ETIMEDOUT) {
+			svc.fwd_gr_timeout++;
+			if (cp.fault.valid)
+				svc.fwd_gr_fault++;
+		}
+		gsp_phys_gr_mme_release_failed(hw_rc);
+	} else {
+		svc.fwd_gr_ok++;
+		gsp_phys_gr_mme_release_complete();
+	}
+	svc_fwd_req.fault = cp.fault;
+	svc_fwd_req.hw_rc = hw_rc;
+	smp_store_release(&svc_fwd_req.done, true);
+	clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+	task = READ_ONCE(svc.task);
+	if (task)
+		wake_up_process(task);
+}
+
 /*
  * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The
  * request already resolved every launch page's guest VA->PA on the service
@@ -38689,6 +38760,15 @@ static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu);
 static void svc_fwd_gr_work(void)
 {
 	/*
+	 * Saved-window release carrier: the slot carries no capture of its
+	 * own; the booter's saved snapshot is the launch. Takes its own
+	 * completion shape (no guest release/QMD word).
+	 */
+	if (svc_fwd_req.gr_release) {
+		svc_fwd_gr_release_work();
+		return;
+	}
+	/*
 	 * Static: the ordered gsp-phys-work queue runs one item at a time and
 	 * the GR page array is ~0.4 KB; keeping it off the stack keeps the
 	 * frame under the kernel's warning bound (same reason as the ops array
@@ -41951,6 +42031,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 	svc_fwd_req.guest = false;
 	svc_fwd_req.gr = true;
 	svc_fwd_req.gr_synthetic = false;
+	svc_fwd_req.gr_release = false;
 	svc_fwd_req.gr_bridge_done = false;
 	/* STALLFREE: fresh request - re-arm the software-completion latch and
 	 * stamp the fill time for the GR-STALL waited_ms accounting. */
@@ -43488,8 +43569,6 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
  * or written. The booter re-validates the live cursor in carry_take(); on any
  * mismatch it cancels and this function is a no-op.
  */
-#define SVC_GR_CARRIER_PUSH_VA	0x6d6d6500ULL	/* unique synthetic id */
-#define SVC_GR_CARRIER_STUB_WAIT	0x20010044u	/* INC 0x110 WAIT_FOR_IDLE */
 
 static bool svc_gr_cand_unoffered_for(u32 h_object)
 {
@@ -43505,10 +43584,10 @@ static bool svc_gr_cand_unoffered_for(u32 h_object)
 }
 
 static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu,
-				   const struct svc_channel *ch)
+				   const struct svc_channel *ch, bool release)
 {
 	u32 h_object = ch->h_object;
-	u32 gen = 0, off = 0, maxa = 0;
+	u32 gen = 0, off = 0, maxa = 0, held = 0, win = 0;
 
 	if (!gsp_phys_gr_forward_enabled() ||
 	    !gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled())
@@ -43516,11 +43595,19 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu,
 	/* One launch owner per channel: never compete with a real candidate. */
 	if (svc_gr_cand_unoffered_for(h_object) || svc_fwd_hold.active)
 		return false;
-	if (!gsp_phys_gr_mme_carry_take(h_object, &gen, &off, &maxa))
-		return false;
-	pr_info("vgsp-gr: MME-CARRY-FIRE h=0x%x gen=%u off=%u/%u push=0x%llx\n",
-		h_object, gen, off, maxa,
-		(unsigned long long)SVC_GR_CARRIER_PUSH_VA);
+	if (release) {
+		if (!gsp_phys_gr_mme_release_take(h_object, &held, &win))
+			return false;
+		pr_info("vgsp-gr: MME-CARRY-FIRE mode=release held=%u win=%u h=0x%x push=0x%llx\n",
+			held, win, h_object,
+			(unsigned long long)SVC_GR_CARRIER_PUSH_VA);
+	} else {
+		if (!gsp_phys_gr_mme_carry_take(h_object, &gen, &off, &maxa))
+			return false;
+		pr_info("vgsp-gr: MME-CARRY-FIRE h=0x%x gen=%u off=%u/%u push=0x%llx\n",
+			h_object, gen, off, maxa,
+			(unsigned long long)SVC_GR_CARRIER_PUSH_VA);
+	}
 	mutex_lock(&svc_fwd_lock);
 	if (svc_fwd_paused ||
 	    test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) {
@@ -43530,7 +43617,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu,
 		 * without rewinding the pack (the next client window continues
 		 * from the current cursor).
 		 */
-		gsp_phys_gr_mme_carry_cancel("request-busy");
+		if (release)
+			gsp_phys_gr_mme_release_cancel("request-busy");
+		else
+			gsp_phys_gr_mme_carry_cancel("request-busy");
 		return false;
 	}
 	memset(&svc_fwd_req, 0, sizeof(svc_fwd_req));
@@ -43542,21 +43632,33 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu,
 	svc_fwd_req.gr_h_object = h_object;
 	svc_fwd_req.gr_push_va = SVC_GR_CARRIER_PUSH_VA;
 	svc_fwd_req.gr_fill_jiffies = jiffies;
-	/*
-	 * The backend's 3D branch is entered by a non-empty stream; the
-	 * two-word stub here is only the request shape (the booter replaces it
-	 * with its own WAIT_FOR_IDLE stub and forces LOAD-ONLY), so this body
-	 * is never executed. No pages, inline records, data records, windows,
-	 * program binds, sem fields or release fields are filled - the memset
-	 * above zeroed them and qmd[] is the all-zero 64-word source the
-	 * submit requires.
-	 */
-	svc_fwd_req.gr_stream_3d = true;
-	svc_fwd_req.gr_stream_off = 0;
-	svc_fwd_req.gr_stream_n = 2;
-	svc_fwd_req.gr_stream_rel = 0;
-	svc_fwd_req.gr_raw[0] = SVC_GR_CARRIER_STUB_WAIT;
-	svc_fwd_req.gr_raw[1] = 0;
+	if (release) {
+		/*
+		 * Saved-window release: the slot carries no capture of its
+		 * own. The work item fills the submit request from the
+		 * booter's saved record (gr_release) - window, pages, inline
+		 * and data all come from that snapshot - and submits it with
+		 * the held CALL runs released.
+		 */
+		svc_fwd_req.gr_release = true;
+	} else {
+		/*
+		 * The backend's 3D branch is entered by a non-empty stream;
+		 * the two-word stub here is only the request shape (the
+		 * booter replaces it with its own WAIT_FOR_IDLE stub and
+		 * forces LOAD-ONLY), so this body is never executed. No
+		 * pages, inline records, data records, windows, program
+		 * binds, sem fields or release fields are filled - the
+		 * memset above zeroed them and qmd[] is the all-zero 64-word
+		 * source the submit requires.
+		 */
+		svc_fwd_req.gr_stream_3d = true;
+		svc_fwd_req.gr_stream_off = 0;
+		svc_fwd_req.gr_stream_n = 2;
+		svc_fwd_req.gr_stream_rel = 0;
+		svc_fwd_req.gr_raw[0] = SVC_GR_CARRIER_STUB_WAIT;
+		svc_fwd_req.gr_raw[1] = 0;
+	}
 	svc_fwd_requests++;
 	if (!svc_fwd_wq) {
 		svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0);
@@ -43564,7 +43666,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu,
 			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
 			svc_fwd_dropped++;
 			mutex_unlock(&svc_fwd_lock);
-			gsp_phys_gr_mme_carry_cancel("queue-alloc");
+			if (release)
+				gsp_phys_gr_mme_release_cancel("queue-alloc");
+			else
+				gsp_phys_gr_mme_carry_cancel("queue-alloc");
 			return false;
 		}
 	}
@@ -43572,7 +43677,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu,
 		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
 		svc_fwd_dropped++;
 		mutex_unlock(&svc_fwd_lock);
-		gsp_phys_gr_mme_carry_cancel("queue-work");
+		if (release)
+			gsp_phys_gr_mme_release_cancel("queue-work");
+		else
+			gsp_phys_gr_mme_carry_cancel("queue-work");
 		return false;
 	}
 	mutex_unlock(&svc_fwd_lock);
@@ -43649,6 +43757,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
 	 */
 	svc_fwd_req.gr = false;
 	svc_fwd_req.gr_synthetic = false;
+	svc_fwd_req.gr_release = false;
 	svc_fwd_req.gr_bridge_done = false;
 	svc_fwd_req.guest = svc_fwd_cand.valid &&
 			    gsp_phys_guest_mem_enabled();
@@ -49921,6 +50030,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 	}
 
 	/*
+	 * Saved-window release carrier (sema_gl_mme_pack_release). After the
+	 * channel loop (a client window decoded in this pass wins and cancels
+	 * the pending release), fire the saved held-call window once the pack
+	 * completed and the settle grace expired. Same gates as the
+	 * continuation carrier, except the FE-busy defer (this launch carries
+	 * a real window, not a LOAD-ONLY carrier).
+	 */
+	{
+		u32 rel_h = 0;
+
+		if (gsp_phys_gr_mme_release_peek(&rel_h)) {
+			int ci;
+
+			for (ci = 0; ci < n; ci++) {
+				struct svc_channel *c = &local[ci];
+
+				if (c->h_object != rel_h || c->engine_type)
+					continue;
+				if (svc_gr_cand_unoffered_for(rel_h))
+					break;
+				if (svc_fwd_hold.active ||
+				    test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state))
+					break;
+				(void)svc_gr_carrier_request(vgpu, c, true);
+				break;
+			}
+		}
+	}
+
+	/*
 	 * c727 host-initiated MME pack continuation. After the channel loop (so
 	 * a client window decoded in this pass can complete the pack first),
 	 * emit the next golden pack part for the armed beef channel. The gates
@@ -49949,7 +50088,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					break;
 				if (gsp_phys_gr_mme_fe_defer_wanted(carry_h, 2))
 					break;
-				(void)svc_gr_carrier_request(vgpu, c);
+				(void)svc_gr_carrier_request(vgpu, c, false);
 				break;
 			}
 		}