vgsp-sema: software-author the GR launch completion release when the physical forward fails (ISSUE-0002)

r51/r52 root evidence: the second session's GR launch was forwarded to the
physical engine; the backend timed out (qmd_rel=5 want=6 after 5 s) while the
hold resolution applied only the *stream-recorded* channel release. The QMD
launch completion semaphore (the word the client polls in its launch-
completion wait) is carried by the GR candidate (gr_rel_va/gr_rel_payload),
not by the release records, and nothing authored it on failure - the client
spun forever in the completion wait and the pump went idle (pushes=1207,
pending=499, crel frozen).

Fix:
- a GR hold waits for the backend's bounded outcome (12 s bound vs the
  backend's 5 s poll) instead of the 500 ms CE copy bound, so the release is
  never authored while the physical submission is still writing guest pages;
- on hold resolution without a successful hardware copy (timeout, abort,
  expiry, or a release0 page the backend could not observe), the QMD release
  is software-applied forward-only (read/compare, absolute payload write) -
  idempotent against a late engine write and never rewinds the tracker;
- a GR capture whose request did not queue (refused/backend busy/hold active)
  authors the same release inline when the CUDA bridge is disabled (the
  bridge job owns it otherwise);
- new work_guard counter gr_sw_rel names how often the fallback authored a
  release, and each write logs its va/payload.

Validated: c1 GREEN, c2 GREEN in the same VM (r54; CE->GR, mail + VADD
STATUS OK). In that run the forward succeeded so gr_sw_rel stayed 0; the
timeout branch is the one r52 proved is reachable.

1 files changed, 96 insertions, 3 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index fd70869..5fd583a 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -716,6 +716,13 @@ struct sema_svc {
 	u64			fwd_gr_timeout;
 	u64			fwd_gr_fault;
 	u64			fwd_gr_bridge;
+	/*
+	 * GR launch completion releases (the QMD-level semaphore the physical
+	 * engine authors on success) that the emulation had to write itself
+	 * because the hardware copy did not deliver them (forward timeout,
+	 * abort, or a release0 page the backend could not observe).
+	 */
+	u64			fwd_gr_sw_rel;
 
 	/* Distinct stalled VA tracking for rate-limited logging. */
 #define SVC_STALLED_MAX		256
@@ -3294,14 +3301,16 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 	    svc.fwd_rel_deferred || svc.fwd_rel_applied ||
 	    svc.fwd_rel_fallback || svc.fwd_rel_pending ||
 	    svc.fwd_rel_overflow || svc.fwd_hold_engaged ||
-	    svc.fwd_hold_timeouts || svc.fwd_hold_canceled)
-		pr_info("sema-svc:   work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu\n",
+	    svc.fwd_hold_timeouts || svc.fwd_hold_canceled ||
+	    svc.fwd_gr_sw_rel)
+		pr_info("sema-svc:   work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu gr_sw_rel=%llu\n",
 			svc.fwd_rel_deferred, svc.fwd_rel_applied,
 			svc.fwd_rel_fallback, svc.fwd_rel_pending,
 			svc.fwd_rel_overflow, svc.fwd_hold_engaged,
 			svc.fwd_hold_timeouts, svc.fwd_hold_canceled,
 			svc.fwd_copy_skipped, svc.fwd_copy_soft,
-			svc.fwd_copy_fail, svc.rel_hold_direct);
+			svc.fwd_copy_fail, svc.rel_hold_direct,
+			svc.fwd_gr_sw_rel);
 	/* show a few entries so mappings can be verified from dmesg */
 	if (gsp_sema_dbg) {
 		u32 shown;
@@ -10364,6 +10373,15 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu)
  * failure / 500 ms timeout / abort, drop on a vanished channel.
  */
 #define SVC_FWD_HOLD_TIMEOUT_MS	500u
+/*
+ * A GR launch hold waits for the physical submission's bounded outcome
+ * (the backend's GR poll is 5 s) before its software fallback fires; the CE
+ * copy bound stays 500 ms. Expiring a GR hold while the request is still in
+ * flight would author the launch completion release before the engine has
+ * finished writing the guest pages, so the client could observe data the
+ * hardware has not produced yet.
+ */
+#define SVC_FWD_GR_HOLD_TIMEOUT_MS	12000u
 
 struct svc_fwd_hold {
 	bool active;
@@ -10537,8 +10555,54 @@ static void svc_fwd_hold_direct_note(void)
 					svc_fwd_hold.rel[i].payload);
 }
 
+/*
+ * Author a GR launch completion release (the QMD-level semaphore the
+ * physical engine writes when the launch completes) in software,
+ * forward-only. Used when a hold resolves without a successful hardware
+ * copy: the backend timed out or aborted, or release0 could not be observed
+ * on the guest page so only the module semaphore carried the outcome.
+ * Without this the guest tracker keeps polling a word the engine never
+ * wrote and the client wedges with the pump idle (ISSUE-0002: the second
+ * session's forwarded launch timed out and the client spun in the launch
+ * completion wait forever). A word already at/ahead of the payload is left
+ * alone - the engine, the CUDA bridge or an earlier fallback authored it.
+ */
+static void svc_gr_sw_release(struct nv_vgpu_device *vgpu, u32 h, u64 va,
+			      u32 payload)
+{
+	u32 cur = 0;
+
+	if (!vgpu || !va || !payload)
+		return;
+	if (!svc_sem_read(vgpu, va, &cur) && cur >= payload)
+		return;
+	if (!svc_sem_write(vgpu, h, va, payload)) {
+		svc.fwd_gr_sw_rel++;
+		pr_info("vgsp-work: GR launch release software-applied h=0x%x va=0x%llx pay=0x%x (no successful hardware copy; forward-only)\n",
+			h, va, payload);
+	}
+}
+
+/*
+ * The held GR request's QMD completion release, if any. The request slot is
+ * stable while a hold is active (svc_fwd_request refuses reuse), so the
+ * svc_fwd_req fields read here belong to this hold's submission.
+ */
+static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu,
+				    const struct svc_channel *ch)
+{
+	if (!svc_fwd_req.gr)
+		return;
+	svc_gr_sw_release(vgpu, ch ? ch->h_object : 0,
+			  svc_fwd_req.gr_rel_va,
+			  svc_fwd_req.gr_rel_payload);
+}
+
 static bool svc_fwd_hold_expired(void)
 {
+	if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done))
+		return time_after(jiffies, svc_fwd_hold.since +
+				  msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS));
 	return time_after(jiffies, svc_fwd_hold.since +
 			  msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS));
 }
@@ -10764,6 +10828,7 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu,
 		shim.engine_type = svc_fwd_hold.engine_type;
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
+		svc_fwd_hold_gr_release(hv, &shim);
 	}
 	if (!(done && svc_fwd_req.hw_rc == 0))
 		svc_fwd_hold_direct_note();
@@ -10868,11 +10933,21 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
 			svc.fwd_relay_fallback++;
 		svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
+		svc_fwd_hold_gr_release(hv, ch);
 		svc_fwd_hold_direct_note();
 		svc.fwd_rel_fallback++;
 		if (!done)
 			svc.fwd_hold_timeouts++;
 	}
+	/*
+	 * GR launch completion release: on a successful hardware copy the
+	 * engine normally authored it; when release0 had no guest page to
+	 * observe (module-semaphore fallback) the guest word is still behind,
+	 * so author it forward-only here as well. No-op when the word already
+	 * carries the payload (success path with a resolved release page).
+	 */
+	if (done && svc_fwd_req.hw_rc == 0)
+		svc_fwd_hold_gr_release(hv, ch);
 	/* Mirror the normal consume tail for the held entry. */
 	svc_replay_note(ch, idx, put);
 	ch->last_get = (idx + 1) % ch->gpfifo_entries;
@@ -13059,6 +13134,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									if (svc_fwd_req.relay)
 										svc.fwd_relay_fallback++;
 									svc_fwd_req.relay = false;
+									/*
+									 * A GR capture whose request
+									 * did not queue (refused,
+									 * backend busy, hold active)
+									 * has no hardware copy coming:
+									 * author its QMD completion
+									 * release forward-only so the
+									 * client's wait is not left to
+									 * a launch that never ran. With
+									 * the CUDA bridge enabled the
+									 * bridge job owns the release.
+									 */
+									if (!ch->engine_type &&
+									    !gsp_cuda_bridge_enabled())
+										svc_gr_sw_release(vgpu,
+												  ch->h_object,
+												  svc_gr_cand.rel_va,
+												  svc_gr_cand.rel_payload);
 								}
 							}
 							/*