vgsp: GR fill hold - absolute deadline + expiry-proceed + dropped-page detail

3 files changed, 230 insertions, 101 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 51310a2..6286d38 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -377,18 +377,23 @@ struct svc_channel {
 				 * even if still unresolved - bounded zeros, never
 				 * a hang); m156: re-held while the window is
 				 * live so a busy forward gets its real retry */
-	unsigned long gr_hold_deadline;	/* GRFIX FIX1: hard expiry for the data-
-				 * stalled hold of gr_hold_dead_idx. The
-				 * m156 re-arm restarts gr_hold_until on
-				 * every pass, so a page that never
-				 * resolves would otherwise keep the
-				 * entry held forever (the r176 stall
-				 * class). Fixed once on the first arm;
-				 * re-arms never move it; at the deadline
-				 * the entry is consumed with the
-				 * historical software release (bounded,
-				 * never a hang). */
-	u32	gr_hold_dead_idx;	/* entry gr_hold_deadline was armed for */
+	unsigned long gr_hold_deadline;	/* GRFIX FIX1b: hard expiry of the data-
+				 * stalled hold for gr_hold_dead_va.
+				 * Fixed once on the first hold of that
+				 * launch push VA and NEVER moved while
+				 * the launch stays held; when it passes
+				 * the launch proceeds (pre-8b058884
+				 * behaviour), so the guest can never
+				 * wait unboundedly. Keyed by the launch
+				 * VA, not the GPFIFO slot: pin-batch
+				 * rewinds re-decode one launch at
+				 * different slot indices (r213:
+				 * entry 107 -> 107 -> 99), which
+				 * re-armed a slot-keyed deadline on
+				 * every pass. */
+	u64	gr_hold_dead_va;	/* launch push VA the bound belongs to */
+	unsigned long gr_hold_dead_since;	/* first-hold stamp (waited_ms) */
+	bool	gr_hold_exp_logged;	/* one GRFILL-EXPIRED line per launch */
 	u32	gr_rehold_idx;	/* m156: slot whose next decode must force a
 				 * fresh launch capture (a held entry is
 				 * re-decoded as a replay, which suppresses
@@ -1271,7 +1276,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 
 		memset(ch, 0, sizeof(*ch));
 		ch->gr_hold_idx = ~0u;	/* m156: no hold armed yet */
-		ch->gr_hold_dead_idx = ~0u;	/* GRFIX: no hard bound armed */
+		ch->gr_hold_dead_va = 0;	/* GRFIX: no launch bound armed */
+		ch->gr_hold_exp_logged = false;
 		ch->gr_rehold_idx = ~0u;
 		ch->gr_nocap_idx = ~0u;	/* m164: no no-capture window armed */
 		ch->session_gen = svc_sess_gen;	/* m176 */
@@ -10424,6 +10430,19 @@ bool gsp_sema_counter_derive = true;
 unsigned gsp_sema_gr_grace_ms = 8000;
 
 /*
+ * GRFIX FIX1b (r213): absolute per-LAUNCH bound (ms) for the data-stalled
+ * hold of svc_gr_data_fill. The identity is the launch push VA, not the
+ * GPFIFO slot: pin-batch rewinds/replays re-decode the same launch at
+ * different slot indices (r213: entry 107 -> 107 -> 99 -> ... on one
+ * channel), so an idx-keyed deadline re-armed on every pass and never
+ * fired - 410+ holds over 216 s, gr_hold_exp=0, client killed at its 240 s
+ * timeout. The deadline is fixed at the first hold of a launch; when it
+ * passes, svc_gr_request proceeds with the launch exactly as the
+ * pre-8b058884 path did (forward best-effort - never block the guest).
+ */
+unsigned gsp_phys_gr_hold_max_ms = 3000;
+
+/*
  * F1-b: GR release-word authorship gate (default 1). While a forwarded GR
  * launch is in flight, the RAM-content sweep must not author the very QMD
  * completion release the backend poll is waiting for: a sweep write could
@@ -12092,6 +12111,9 @@ struct svc_fwd_req {
 	bool gr_bridge_done;
 	bool gr_page_overflow;
 	u32 gr_h_object;
+	u32 gr_dropped;		/* GRFIX: pages the fill's last pass could not
+				 * resolve (0 = launch fully resolved); read
+				 * by the absolute-bound expiry log */
 	u64 gr_push_va;
 	u32 gr_push_len;
 	u32 gr_push_n;
@@ -13400,6 +13422,10 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
 	ch->last_get = (idx + 1) % ch->gpfifo_entries;
 	ch->rescue_fail = 0;
 	ch->defer_since = 0;
+	/* GRFIX FIX1b: this entry is consumed; its per-launch absolute-bound
+	 * record resets so a later launch (slot/VA reuse) arms a fresh one. */
+	ch->gr_hold_dead_va = 0;
+	ch->gr_hold_exp_logged = false;
 	ch->submissions++;
 	svc.gpfifo_entries_done++;
 	if (ch->engine_type && ch->expected_sem_payload)
@@ -14192,12 +14218,17 @@ static bool svc_gr_data_stalled;	/* m150: high data args unresolved */
  */
 static bool svc_gr_busy_defer;
 static u64 svc_gr_hold_log;	/* m150: bounded GR-hold proof lines */
+static u64 svc_gr_drop_log_va;	/* GRFIX: launch whose dropped-page detail
+				 * (GRFILL-DROP) was already logged - the
+				 * detail prints once per launch episode,
+				 * not once per ~100 ms re-decode */
 
 /* m150: re-arm both fill diagnostics for each client session. */
 static void svc_gr_diag_reset(void)
 {
 	svc_gr_data_log = 0;
 	svc_gr_src_log = 0;
+	svc_gr_drop_log_va = 0;
 }
 
 /*
@@ -14312,6 +14343,35 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va)
 	}
 }
 
+/*
+ * GRFIX FIX1b (r213): absolute per-launch hold bound. The m150/m156 data
+ * hold had no working bound: the 8b058884 deadline was keyed by the GPFIFO
+ * slot index, and a pin-batch rewind/replay re-decodes the same launch push
+ * at different slots (r213: entry 107 -> 107 -> 99 on one channel, other
+ * passes at yet other indices), so "first arm" re-armed on every pass; the
+ * launch held for 216+ s (410+ GRFILL-HOLD lines) until the client's 240 s
+ * timeout killed it (gr_hold_exp=0). The identity is now the launch push VA:
+ * the first hold of a VA arms an ABSOLUTE deadline that nothing moves while
+ * the launch is held. Returns true when the bound has passed (caller must
+ * let the launch proceed); false leaves the record armed.
+ */
+static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
+{
+	u32 cap_ms = gsp_phys_gr_hold_max_ms;
+
+	if (!launch_va)
+		launch_va = 1;	/* anonymous launch: still absolutely bound */
+	if (!cap_ms)
+		cap_ms = 3000;
+	if (ch->gr_hold_dead_va != launch_va) {
+		ch->gr_hold_dead_va = launch_va;
+		ch->gr_hold_dead_since = jiffies;
+		ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms);
+		ch->gr_hold_exp_logged = false;
+	}
+	return time_after_eq(jiffies, ch->gr_hold_deadline);
+}
+
 static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 {
 	u64 wins[16];
@@ -14348,10 +14408,25 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 	 * first-page guard below, so a QMD is never forwarded with a data
 	 * pointer left as an unresolvable guest VA (the zero-store class)
 	 * and the bounce window is never served stale for this launch.
+	 * FIX1b: the hold carries an absolute per-launch bound
+	 * (gsp_phys_gr_hold_max_ms) - when it passes the launch proceeds.
 	 */
 	u32 dropped_pages = 0;
+	/*
+	 * GRFIX FIX1c (r213): detail for the first (up to 4) dropped pages,
+	 * so the cycle records WHICH VAs and which candidate class dropped.
+	 * There is no read/write direction anywhere in the candidate or
+	 * data[] structures (both are va/pa/reloc_va only), so kind is the
+	 * truthful candidate origin: 'a' = argument page (c[0x0] param /
+	 * inline), 'b' = const-bank page; idx is the candidate index (the
+	 * same index the per-candidate probe lines use).
+	 */
+	u64 drop_va[4];
+	char drop_kind[4];
+	u32 drop_idx[4];
 
 	svc_fwd_req.gr_n_data = 0;
+	svc_fwd_req.gr_dropped = 0;
 	svc_fwd_req.gr_n_data_wins = 0;
 	svc_gr_data_stalled = false;
 	memset(is_arg, 0, sizeof(is_arg));
@@ -14586,10 +14661,20 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 						 * span) for the launch-level
 						 * hold; the span still stops
 						 * here exactly as before.
+						 * FIX1c: capture the first 4
+						 * (VA + candidate origin +
+						 * candidate index) for the
+						 * GRFILL-DROP detail.
 						 */
 						if (p == 0 &&
 						    svc_fwd_req.gr_n_data == before)
 							first_unresolved = true;
+						if (dropped_pages < 4) {
+							drop_va[dropped_pages] = va;
+							drop_kind[dropped_pages] =
+								is_arg[w] ? 'a' : 'b';
+							drop_idx[dropped_pages] = w;
+						}
 						dropped_pages++;
 						break;
 					}
@@ -14740,10 +14825,28 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 	 */
 	svc_gr_data_stalled = (dropped_pages > 0) ||
 			      (n_arg_hi > 0 && n_arg_hi_res == 0);
-	if (dropped_pages > 0)
-		pr_info_ratelimited("vgsp-sema: vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n",
+	svc_fwd_req.gr_dropped = dropped_pages;
+	if (dropped_pages > 0) {
+		pr_info_ratelimited("vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n",
 			svc_fwd_req.gr_push_va, dropped_pages,
 			svc_fwd_req.gr_n_data);
+		/*
+		 * GRFIX FIX1c: name the dropped pages once per launch
+		 * episode (the fill re-runs every pass for a held launch;
+		 * the marker makes the detail a first-pass-only record).
+		 * kind 'a' = argument-page candidate, 'b' = bank candidate;
+		 * idx = candidate index (same indexing as the probe lines).
+		 */
+		if (svc_gr_drop_log_va != svc_fwd_req.gr_push_va) {
+			u32 dl;
+
+			svc_gr_drop_log_va = svc_fwd_req.gr_push_va;
+			for (dl = 0; dl < 4 && dl < dropped_pages; dl++)
+				pr_info("vgsp-gr: GRFILL-DROP va=0x%llx kind=%c idx=%u\n",
+					drop_va[dl], drop_kind[dl],
+					drop_idx[dl]);
+		}
+	}
 	if (svc_gr_data_stalled)
 		pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), %u resolved; %u dropped page(s) -> launch held for a bounded retry\n",
 			n_cand, svc_fwd_req.gr_n_data, n_arg_hi, n_arg_hi_res,
@@ -14997,6 +15100,40 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 	 * expires the entry is consumed exactly as before.
 	 */
 	if (svc_gr_data_stalled) {
+		u64 lva = svc_fwd_req.gr_push_va;
+		/*
+		 * The channel is the pump's mutable live bookkeeping object
+		 * (the const view exists for the read-only gates); these
+		 * record fields are the same ones the pump's hold writes,
+		 * on this same service thread.
+		 */
+		struct svc_channel *mch = (struct svc_channel *)ch;
+
+		/*
+		 * GRFIX FIX1b (r213): the ABSOLUTE per-launch bound. The
+		 * first hold of this push VA arms a fixed deadline
+		 * (svc_gr_hold_bound_expired) that nothing re-arms; once it
+		 * passes, the launch PROCEEDS exactly as the pre-8b058884
+		 * path - the candidate is consumed below and forwarded
+		 * best-effort - so the guest is never blocked by a page that
+		 * never resolves. r213: a slot-keyed deadline was re-armed
+		 * on every re-decode (the same launch is replayed at
+		 * different GPFIFO slots after rewinds) and held one launch
+		 * for 216+ s until the client's 240 s timeout killed it.
+		 */
+		if (svc_gr_hold_bound_expired(mch, lva)) {
+			svc.fwd_gr_hold_exp++;
+			if (!mch->gr_hold_exp_logged) {
+				mch->gr_hold_exp_logged = true;
+				pr_info("vgsp-gr: GRFILL-EXPIRED launch=0x%llx dropped=%u waited_ms=%u\n",
+					lva, svc_fwd_req.gr_dropped,
+					jiffies_to_msecs(jiffies -
+							 mch->gr_hold_dead_since));
+			}
+			svc_gr_data_stalled = false;	/* proceed: forward */
+		}
+	}
+	if (svc_gr_data_stalled) {
 		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
 		mutex_unlock(&svc_fwd_lock);
 		svc.fwd_gr_refused++;
@@ -15816,11 +15953,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				}
 			}
 			bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS);
-			/* GRFIX FIX1: a recycled slot in the next put epoch
-			 * gets a fresh hard-bound episode for its hold;
-			 * without this a wrapped slot would inherit a stale
-			 * expired deadline and never retry. */
-			ch->gr_hold_dead_idx = ~0u;
+			/* GRFIX FIX1b: a recycled slot in the next put epoch
+			 * gets a fresh absolute-bound record; without this a
+			 * wrapped slot would inherit a stale expired launch
+			 * record. */
+			ch->gr_hold_dead_va = 0;
+			ch->gr_hold_exp_logged = false;
 		}
 		put_prev_wit = ch->put_prev;
 		ch->put_prev = put;
@@ -16404,10 +16542,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								fwd_guest_queued =
 									svc_fwd_request(vgpu, ch);
 								if (fwd_guest_queued &&
-								    !ch->engine_type) {
+								    !ch->engine_type)
 									ch->gr_hold_idx = ~0u;
-									ch->gr_hold_dead_idx = ~0u;
-								}
 								/*
 								 * m159: a busy refusal (forward
 								 * in flight / release hold
@@ -16519,23 +16655,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								}
 								/*
 								 * m150: the GR fill held this
-								 * launch because its high
-								 * argument pointers do not
-								 * translate yet. Leave the
-								 * entry unconsumed (do not
-								 * write GP_GET) so the pump
-								 * re-captures and retries it;
-								 * one bounded window per entry,
-								 * then fall through and consume
-								 * exactly as before. The slot's
-								 * identity makes the hold
-								 * one-shot per entry: after
-								 * expiry the same entry is
-								 * never re-held.
+								 * launch because it carries
+								 * data pages that do not
+								 * resolve yet. Leave the entry
+								 * unconsumed (do not write
+								 * GP_GET) so the pump
+								 * re-captures and retries it.
+								 * GRFIX FIX1b (r213): the hold
+								 * is bounded ABSOLUTELY per
+								 * launch push VA
+								 * (svc_gr_hold_bound_expired
+								 * fixes the deadline on the
+								 * first hold and nothing
+								 * re-arms it; svc_gr_request
+								 * enforces it and forwards the
+								 * launch once it passes). The
+								 * extra check here is the
+								 * defence for a stale stalled
+								 * verdict when the request
+								 * path did not run this pass -
+								 * past the bound the entry
+								 * falls through to the
+								 * historical consume.
 								 */
 								if (!fwd_guest_queued &&
 								    !ch->engine_type &&
 								    svc_gr_data_stalled &&
+								    !svc_gr_hold_bound_expired(ch,
+									    svc_fwd_req.gr_push_va) &&
 								    (ch->gr_hold_idx != (u32)idx ||
 								     time_before(jiffies,
 										 ch->gr_hold_until))) {
@@ -16543,76 +16690,35 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 										gsp_sema_gr_grace_ms : 8000;
 
 									/*
-									 * GRFIX FIX1 hard bound: the
-									 * m156 re-arm below restarts
-									 * gr_hold_until on every
-									 * pass, so a page that never
-									 * resolves would keep this
-									 * entry held forever (the
-									 * r176 stall class). Fix the
-									 * deadline on the first arm;
-									 * re-arms never move it.
+									 * m156: arm the forced
+									 * re-capture so the next
+									 * decode of this entry
+									 * rebuilds the GR candidate
+									 * (a replay decode would
+									 * suppress it) and the
+									 * deferred launch is
+									 * forwarded for real once
+									 * the in-flight work has
+									 * completed.
 									 */
-									if (ch->gr_hold_dead_idx !=
-									    (u32)idx) {
-										ch->gr_hold_dead_idx =
-											(u32)idx;
-										ch->gr_hold_deadline =
-											jiffies +
-											msecs_to_jiffies(
-												hold_ms);
-									}
-									if (time_after_eq(jiffies,
-											  ch->gr_hold_deadline)) {
-										/*
-										 * Bound reached: fall
-										 * through to the
-										 * historical consume
-										 * (the software release
-										 * is applied below) - the
-										 * guest never stalls.
-										 */
-										svc.fwd_gr_hold_exp++;
-										pr_info_ratelimited("sema-svc: GR data-hold hard bound h=0x%x entry=%u (%u ms); consuming with software release\n",
+									ch->gr_rehold_idx =
+										(u32)idx;
+									ch->gr_hold_idx = (u32)idx;
+									ch->gr_hold_until = jiffies +
+										msecs_to_jiffies(hold_ms);
+									svc.fwd_gr_defer++;
+									if (svc_gr_hold_log < 16) {
+										svc_gr_hold_log++;
+										pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n",
 											ch->h_object,
 											(u32)idx,
+											svc_gr_data_stalled ?
+												"data args unresolved" :
+												"forward busy (m156)",
 											hold_ms);
-									} else {
-										/*
-										 * m156: arm the
-										 * forced re-capture so
-										 * the next decode of
-										 * this entry rebuilds
-										 * the GR candidate (a
-										 * replay decode would
-										 * suppress it) and the
-										 * deferred launch is
-										 * forwarded for real
-										 * once the in-flight
-										 * work has completed.
-										 */
-										ch->gr_rehold_idx =
-											(u32)idx;
-										ch->gr_hold_idx =
-											(u32)idx;
-										ch->gr_hold_until =
-											jiffies +
-											msecs_to_jiffies(
-												hold_ms);
-										svc.fwd_gr_defer++;
-										if (svc_gr_hold_log < 16) {
-											svc_gr_hold_log++;
-											pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n",
-												ch->h_object,
-												(u32)idx,
-												svc_gr_data_stalled ?
-													"data args unresolved" :
-													"forward busy (m156)",
-												hold_ms);
-										}
-										kfree(pb);
-										break;
 									}
+									kfree(pb);
+									break;
 								}
 								/*
 								 * P6b wave 1.5: when
@@ -17381,6 +17487,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			ch->gr_nocap_va = 0;
 			ch->unread_va = 0;
 			ch->unread_until = 0;
+			/* GRFIX FIX1b: the entry is consumed; the per-launch
+			 * absolute-bound record resets with it. */
+			ch->gr_hold_dead_va = 0;
+			ch->gr_hold_exp_logged = false;
 
 			/* Publish progress the way the GPU would. */
 			g = ch->last_get;
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index e79064c..87b281b 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -268,6 +268,15 @@ extern unsigned gsp_sema_push_hold_ms;
 extern unsigned gsp_sema_gr_grace_ms;
 
 /*
+ * GRFIX FIX1b (r213): absolute per-launch bound (ms) for the GR
+ * data-stalled hold. Keyed by the launch push VA (slot indices are not
+ * stable across pin-batch rewinds); first hold fixes the deadline, nothing
+ * re-arms it, and when it passes the launch proceeds (the pre-8b058884
+ * behaviour) so the guest can never wait unboundedly.
+ */
+extern unsigned gsp_phys_gr_hold_max_ms;
+
+/*
  * m164 CLASS-2: hold window (ms) for a GR launch push whose decode saw QMD
  * words but built no launch candidate; 0 disables the hold.
  */
diff --git a/kernel/main.c b/kernel/main.c
index 60bab26..ae9f262 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2771,6 +2771,16 @@ module_param_named(sema_gr_grace_ms, gsp_sema_gr_grace_ms, uint, 0644);
 MODULE_PARM_DESC(sema_gr_grace_ms, "GR pending-entry grace window in ms (default 8000)");
 
 /*
+ * GRFIX FIX1b (r213): absolute per-launch bound in ms for the GR fill
+ * data-stalled hold (the FIX1 page-drop hold). The deadline is keyed by the
+ * launch push VA, fixed on the first hold and never re-armed; when it passes
+ * the launch proceeds (forwarded best-effort) so a never-resolving page can
+ * never block the guest. 0 keeps the 3000 ms default.
+ */
+module_param_named(gsp_phys_gr_hold_max_ms, gsp_phys_gr_hold_max_ms, uint, 0644);
+MODULE_PARM_DESC(gsp_phys_gr_hold_max_ms, "Absolute GR fill hold bound in ms, keyed by launch push VA (default 3000; launch proceeds when it passes)");
+
+/*
  * F1-b: while a forwarded GR launch is in flight, defer (do not apply) a
  * RAM-sweep release whose VA is that launch's QMD completion word, so the
  * backend poll can only break on an engine-authored ack. Deferred releases