vgsp-sema r548: never consume a GR launch whose forward is still owed (r547 saxpy stall)

Live evidence (r547, 16+ min stuck victory_saxpy): both launches were
captured, deferred by the r541 (unexecuted device write) and r543
(partial argument set) guards, and left as unoffered candidates - then
every later pass re-decoded the entries as replays (LAUNCH-SKIP;
already captured=1) and the consume path dropped them with no GR LAUNCH
and no GR-COMPLETE-SW for the client, whose cudaDeviceSynchronize waited
forever (client-side strace: RM-control poll loop, libcuda condvar
500 ms retimer; host log: consume idx=105/106 every ~2 s, releases never
authored for rel=0x204827f7c).

Three coordinated changes:
 - svc_gr_arg_defer_ok: per-VA latch table (8 slots) instead of a single
   slot. Two interleaved launches made the single slot ping-pong (each
   call saw the other VA and re-deferred both forever); each VA now gets
   exactly one deferral inside the 30 s window and the next sighting
   proceeds, so the guard delays without ever hanging.
 - svc_decode_stream: a fresh decode pass no longer wipes a valid,
   never-offered candidate (a deferred forward is not stale). The
   capture path already rotates to a free candidate-ring slot.
 - pump no-capture hold: fires for a REPLAY whose push VA still has an
   unoffered candidate in the ring (owed forward), forcing a fresh
   capture + request per bounded window; and when the window expires
   with the forward still owed, svc_gr_owed_resolve authors the QMD
   completion release from that candidate and marks it consumed, so an
   entry is never consumed without a completion (no late double-run).

New counter gr_owed in the sema-svc drain line; hold log carries owed=.

1 files changed, 116 insertions, 16 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index c575379..bb52358 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -971,6 +971,9 @@ struct sema_svc {
 						 * words but built no candidate */
 	u64			gr_nocap_consumed;	/* m164: no-capture windows that
 						 * expired and consumed the entry */
+	u64			gr_owed_resolved;	/* r548: unwound deferred launches
+						 * software-completed at hold expiry
+						 * instead of consumed silently */
 	u64			gpfifo_starved;		/* entries consumed without a
 						 * deliverable exact slot */
 	u64			counter_rel_applied;	/* releases to registered counters */
@@ -4705,12 +4708,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued,
 		svc.releases_faked,
 		svc.releases_recovered, svc.counter_rel_applied);
-	pr_info("sema-svc:   drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu gr_swrel=%llu gr_wedge=%llu pend_ovf=%llu\n",
+	pr_info("sema-svc:   drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_owed=%llu gr_hold_exp=%llu gr_swrel=%llu gr_wedge=%llu pend_ovf=%llu\n",
 		svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded,
 		svc.releases_faked, svc.releases_recovered,
 		svc.ops_once_skipped, svc.ops_reident,
 		svc.opdef_held, svc.opdef_applied, svc.opdef_expired,
 		svc.gr_nocap_held, svc.gr_nocap_consumed,
+		svc.gr_owed_resolved,
 		svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel,
 		svc.gr_stale_wedge, svc_pending_ovf);
 	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n",
@@ -9779,8 +9783,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	 * capture: the pump consumes a capture immediately after the pass that
 	 * built it, so anything still here is stale. Replay/rescue decodes
 	 * must not wipe a capture the pump has not consumed yet.
-	 */
-	if (!replay && trusted && ch->engine_type == 0)
+	 *
+	 * r548: a valid, never-offered candidate is NOT stale - it is a launch
+	 * whose forward was DEFERRED (r541/r543 guards, backend busy): the
+	 * pump deliberately left it for a bounded retry. Wiping it here let
+	 * the retry passes re-decode the entry as a replay (candidate already
+	 * captured) and then consume it without any forward or completion
+	 * (the r547 saxpy stall: both launches deferred, ring wiped by an
+	 * interleaved fresh decode, zero GR LAUNCH / GR-COMPLETE-SW for 16+
+	 * minutes). Keep it; the capture path (svc_gr_cand_capture) rotates to
+	 * a free ring slot when a different launch needs the current one.
+	 */
+	if (!replay && trusted && ch->engine_type == 0 &&
+	    (!svc_gr_cand.valid || svc_gr_cand.offered))
 		memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
 	svc_fwd_rel_n = 0;
 	svc_fwd_defer_armed = false;
@@ -20514,23 +20529,97 @@ static void svc_handle_map_add(void);
  * ~1-2 s in the healthy case); the same push VA seen again inside the
  * window proceeds through the historical forward. A re-used ring slot after
  * the window gets its own deferral.
+ *
+ * r548: the latch is a bounded per-VA table, not a single slot. A client
+ * with TWO in-flight launches (victory_saxpy r547: push VAs A and B
+ * alternating re-decodes) made the single slot ping-pong: every call saw
+ * "a different VA than last time" and deferred again - both launches
+ * deferred forever, the client's cudaDeviceSynchronize never completed
+ * (held launches, zero GR LAUNCH lines, zero GR-COMPLETE). The table gives
+ * each VA exactly one deferral inside the window; the second sighting of
+ * any VA proceeds (bounded, cannot hang), interleaved launches included.
  */
 #define SVC_GR_ARG_DEFER_MS	30000u
-static u64 svc_gr_arg_last_va;
-static unsigned long svc_gr_arg_last_j;
+#define SVC_GR_ARG_DEFER_SLOTS	8
+static u64 svc_gr_arg_defer_va[SVC_GR_ARG_DEFER_SLOTS];
+static unsigned long svc_gr_arg_defer_j[SVC_GR_ARG_DEFER_SLOTS];
+static u32 svc_gr_arg_defer_pos;
 
 static bool svc_gr_arg_defer_ok(u64 push_va)
 {
-	if (push_va == svc_gr_arg_last_va &&
-	    time_before(jiffies, svc_gr_arg_last_j +
-			msecs_to_jiffies(SVC_GR_ARG_DEFER_MS)))
-		return false;
-	svc_gr_arg_last_va = push_va;
-	svc_gr_arg_last_j = jiffies;
+	u32 i;
+
+	for (i = 0; i < SVC_GR_ARG_DEFER_SLOTS; i++) {
+		if (svc_gr_arg_defer_va[i] == push_va &&
+		    time_before(jiffies, svc_gr_arg_defer_j[i] +
+				msecs_to_jiffies(SVC_GR_ARG_DEFER_MS)))
+			return false;	/* deferred once already: forward */
+	}
+	i = svc_gr_arg_defer_pos++ % SVC_GR_ARG_DEFER_SLOTS;
+	svc_gr_arg_defer_va[i] = push_va;
+	svc_gr_arg_defer_j[i] = jiffies;
 	return true;
 }
 
 /*
+ * r548: is a forward still OWED for this push VA - a captured, never
+ * offered candidate sitting in the candidate ring? The r547 saxpy stall:
+ * both launches were deferred by the r541/r543 guards, the candidate ring
+ * held them as unoffered, and the later passes re-decoded the entries as
+ * REPLAYS (LAUNCH-SKIP; already captured=1). The replay consume then
+ * dropped the entries without forwarding them and without authoring any
+ * completion (no GR LAUNCH, no GR-COMPLETE-SW) - the client's
+ * cudaDeviceSynchronize waited forever while the pump re-walked the ring.
+ * The pump's no-capture hold must therefore also fire for a replay whose
+ * forward is owed, forcing one fresh capture + request per bounded window.
+ */
+static bool svc_gr_cand_push_owed(u64 push_va)
+{
+	u32 i;
+
+	if (!push_va)
+		return false;
+	for (i = 0; i < SVC_GR_CAND_RING; i++)
+		if (svc_gr_cand_ring[i].valid &&
+		    !svc_gr_cand_ring[i].offered &&
+		    svc_gr_cand_ring[i].push_va == push_va)
+			return true;
+	return false;
+}
+
+/*
+ * r548: last-resort resolution of an owed forward when the bounded
+ * no-capture hold expires without the launch ever being submitted - the
+ * client's completion word must not be left to a launch that never ran.
+ * Author the candidate's QMD release in software (forward-only) and mark
+ * the candidate consumed so a later pass can never forward it late onto
+ * refilled buffers (the r59 double-run class). Returns true when a pending
+ * candidate for this channel+push VA was resolved.
+ */
+static bool svc_gr_owed_resolve(struct nv_vgpu_device *vgpu,
+				const struct svc_channel *ch, u64 push_va)
+{
+	u32 i;
+
+	if (!push_va)
+		return false;
+	for (i = 0; i < SVC_GR_CAND_RING; i++) {
+		struct svc_gr_cand_rec *c = &svc_gr_cand_ring[i];
+
+		if (!c->valid || c->offered ||
+		    c->push_va != push_va ||
+		    c->h_object != ch->h_object)
+			continue;
+		svc_gr_sw_release(vgpu, ch->h_object, c->rel_va,
+				  c->rel_payload);
+		c->offered = true;
+		svc.gr_owed_resolved++;
+		return true;
+	}
+	return false;
+}
+
+/*
  * r541: does any of the filled launch's data pages carry an unexecuted
  * device write (see svc_dstpend_note)? Runs on the service thread right
  * after the fill, so svc_fwd_req.gr_data holds this launch's records.
@@ -22917,7 +23006,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							    gsp_phys_gr_forward_enabled() &&
 							    (svc_gr_qmd_seen > 0 ||
 							     all_zero) &&
-							    !entry_replay &&
+							    (!entry_replay ||
+							     svc_gr_cand_push_owed(va)) &&
 							    !(svc_gr_cand.valid &&
 							      svc_gr_cand.h_object ==
 									ch->h_object &&
@@ -22949,11 +23039,13 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									svc.gr_nocap_held++;
 									if (svc_gr_nocap_log < 16) {
 										svc_gr_nocap_log++;
-										pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d qmd_seen=%u (forcing capture; window %u ms)\n",
+										pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d owed=%d qmd_seen=%u (forcing capture; window %u ms)\n",
 											ch->h_object,
 											(u32)idx, va,
 											len,
 											ch->gr_nocap_replay,
+											svc_gr_cand_push_owed(va) ?
+												1 : 0,
 											svc_gr_qmd_seen,
 											gsp_sema_gr_nocap_hold_ms);
 									}
@@ -22963,11 +23055,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								/* Window expired: consume
 								 * exactly as the pre-change
 								 * tail does, with a proof
-								 * line. */
+								 * line. r548: a forward
+								 * still owed at expiry is
+								 * software-completed first so
+								 * the entry is never consumed
+								 * without a completion. */
+								int owed_done =
+									svc_gr_owed_resolve(vgpu, ch, va);
+
 								svc.gr_nocap_consumed++;
-								pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u (consuming without a launch)\n",
+								pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u owed_resolved=%d (consuming without a launch)\n",
 									ch->h_object,
-									(u32)idx, va, len);
+									(u32)idx, va, len,
+									owed_done);
 							}
 							}
 							/*