gr/sema: r420 pause completion deadlines under the backend wedge-quiet hold

The r419 soak reproduced the user's mid-boot degradation with the wedge fix
in tree: victory_matmul printed bad=65536 while its launch was held by the
booter's WEDGE-QUIET window - the client was released ~115 s before the
post-hold accept. Root cause: two sema-service deadlines are shorter than
the deliberate quiet hold and fire under it, authoring the launch's
completion release (and applying its recorded releases) while the engine has
not run and the copy-out has not happened:

  - the 12 s GR release hold (SVC_FWD_GR_HOLD_TIMEOUT_MS) expiry
    (GR-STALL-fwd-hold ... releases + QMD word software-applied), and
  - the 3 s per-launch consumption bound (gsp_phys_gr_hold_max_ms) firing
    the busy-defer consume-with-software-release path, whose inline apply
    authors the capture's QMD release (GR-REL-AUTHOR src=hold).

Evidence (r419 live): push be94 captured 131333.687, WEDGE-QUIET holds
40 s + 80 s until 131345.598, accept 131346.552 - while the sema hold
expired at 131230.978 (GR-STALL-fwd-hold waited_ms=12032) and the inline
apply authored va=0x204027f7c pay=6 at 131341.734 (h=0x5c00001f). The
client's verify completed on the authored word and read stale data.

Fix: the booter publishes gsp_phys_gr_quiet_hold_active() exactly around
the quiet msleep (bounded by construction). The sema service pauses under
it: svc_fwd_hold_expired() pushes the GR deadline (bounded by a 600 s
defence-in-depth cap, announced once per hold as GR-HOLD-PAUSE);
svc_gr_hold_bound_expired() pushes the per-launch bound
(GR-HOLD-BOUND-PAUSE); and the busy-defer one-shot window re-holds while
the flag is set so the consume-with-sw-release path cannot run during a
quiet hold. The held launch is then forwarded and copied out after the
hold, and its completion is authored only on the hold's done edge - after
the data write-back.

3 files changed, 148 insertions, 5 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index d20e042..f7ad153 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -16324,6 +16324,29 @@ static u64 gsp_phys_gr_wedge_heals;
  * either heals (engine writes, client data correct) or is loudly classified.
  */
 static bool gsp_phys_gr_wedge_episode;
+/*
+ * r420: true exactly while a WEDGE-QUIET hold sleeps with NO submission
+ * outstanding. The hold is a deliberate, bounded wait for the sticky wedge to
+ * clear (r410/r411); during it the in-flight forwarded request is NOT done.
+ * The sema service's completion deadlines - the 12 s GR release hold
+ * (SVC_FWD_GR_HOLD_TIMEOUT_MS) and the 3 s per-launch consumption bound
+ * (gsp_phys_gr_hold_max_ms) - are shorter than this hold, so without the
+ * publish they fire UNDER it and author the launch's release word before the
+ * post-hold copy-out: the client then verifies stale data (r419 soak:
+ * `victory_matmul bad=65536` while its launch was held; the authoring paths
+ * are GR-STALL-fwd-hold / the busy-defer inline software release). The
+ * accessor lets both deadlines pause instead; the flag is bounded by the
+ * msleep itself and by the pause caps on the sema side. Single-inflight
+ * submit path, so a plain static.
+ */
+static bool gsp_phys_gr_quiet_hold_on;
+
+bool gsp_phys_gr_quiet_hold_active(void)
+{
+	return READ_ONCE(gsp_phys_gr_quiet_hold_on);
+}
+EXPORT_SYMBOL_GPL(gsp_phys_gr_quiet_hold_active);
+
 /* m86: program-page override blob (see gsp-phys-gr-override). */
 static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE];
 static u32 gsp_phys_gr_override_len;
@@ -20579,7 +20602,22 @@ gr_page_unmapped:
 						quiet_ms, nw_attempts,
 						req->push_va,
 						READ_ONCE(gsp_phys_gr_nw_streak));
+					/*
+					 * r420: publish the deliberate hold so
+					 * the sema service's completion
+					 * deadlines (the 12 s GR release hold
+					 * and the 3 s per-launch consumption
+					 * bound) pause instead of authoring
+					 * this launch's release word under it -
+					 * the client must not be released
+					 * before the post-hold copy-out
+					 * (gsp_phys_gr_quiet_hold_active).
+					 */
+					WRITE_ONCE(gsp_phys_gr_quiet_hold_on,
+						   true);
 					msleep(quiet_ms);
+					WRITE_ONCE(gsp_phys_gr_quiet_hold_on,
+						   false);
 					quiet_used = quiet_ms;
 					pr_warn("vgsp-gr: WEDGE-QUIET hold done (push=0x%llx); lanes re-armed by the retry below\n",
 						req->push_va);
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index d0c5762..42729d8 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -436,6 +436,16 @@ struct gsp_phys_gr_submit {
  * whose boot submission proved out (rmcall_gr_exec_ok). */
 bool gsp_phys_backend_gr_ready(void);
 
+/*
+ * r420: true exactly while the GR submit path sleeps in a WEDGE-QUIET hold
+ * (a deliberate, bounded wait for the sticky wedge to clear). The in-flight
+ * forwarded request is NOT done during it, so the sema service's completion
+ * deadlines (the 12 s GR release hold and the 3 s per-launch consumption
+ * bound) pause while this returns true instead of authoring the launch's
+ * release word under the hold (the r419 mid-boot stale-data escape).
+ */
+bool gsp_phys_gr_quiet_hold_active(void);
+
 int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				     struct gsp_phys_backend_copy_result *out);
 
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 0354831..6585aaf 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -15436,6 +15436,19 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu)
  * hardware has not produced yet.
  */
 #define SVC_FWD_GR_HOLD_TIMEOUT_MS	12000u
+/*
+ * r420: hard cap on the WEDGE-QUIET deferral of the GR hold deadline. The
+ * backend's quiet hold is a bounded msleep (its own caps: <=240 s per hold,
+ * two holds per launch), so the pause can only last a few minutes; the cap is
+ * a defence-in-depth bound so a stuck quiet-hold flag can never defer the
+ * client's completion indefinitely.
+ */
+#define SVC_FWD_GR_HOLD_PAUSE_MAX_MS	600000u
+/*
+ * r420: same defence for the per-launch consumption bound pause in
+ * svc_gr_hold_bound_expired().
+ */
+#define SVC_GR_QUIET_PAUSE_MAX_MS	600000u
 
 struct svc_fwd_hold {
 	bool active;
@@ -15471,9 +15484,9 @@ struct svc_fwd_hold {
 	/*
 	 * P6c completion relay: the request selected record relay_index and
 	 * the hardware copy was asked to author it into the guest-visible
-	 * semaphore page. On the success edge the hold re-reads the guest
-	 * word and, when it carries the payload, skips that one record's
-	 * software apply. Cleared with the hold.
+	 * semaphore page. On the success edge the hold re-reads the guest word
+	 * and, when it carries the payload, skips that one record's software
+	 * apply. Cleared with the hold.
 	 */
 	bool relay_armed;
 	u8 relay_index;
@@ -15486,6 +15499,18 @@ struct svc_fwd_hold {
 	struct svc_fwd_rel rel[SVC_FWD_REL_MAX];
 	u64 last_rel_va;
 	unsigned long since;
+	/*
+	 * r420: WEDGE-QUIET deferral state. While the backend deliberately
+	 * holds the in-flight launch in its bounded quiet window
+	 * (gsp_phys_gr_quiet_hold_active), the GR deadline is pushed forward
+	 * (paused_until) so the hold cannot author the launch's release word
+	 * under the hold - the client would then verify data the post-hold
+	 * copy-out has not written yet (the r419 mid-boot escape). The
+	 * deferral is itself bounded by SVC_FWD_GR_HOLD_PAUSE_MAX_MS from
+	 * `since` and is announced once per hold.
+	 */
+	unsigned long paused_until;
+	bool pause_logged;
 };
 
 static struct svc_fwd_hold svc_fwd_hold;
@@ -16288,9 +16313,38 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu)
 
 static bool svc_fwd_hold_expired(void)
 {
-	if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done))
+	if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) {
+		/*
+		 * r420: while the backend deliberately holds the in-flight
+		 * launch in its bounded WEDGE-QUIET window, the GR deadline
+		 * must not fire - expiring here authors the launch's release
+		 * word (svc_fwd_hold_gr_release) and applies the recorded
+		 * releases before the post-hold copy-out, so the client
+		 * verifies stale data (the r419 soak's mid-boot
+		 * "matmul fails after time"). Pause instead, bounded by
+		 * SVC_FWD_GR_HOLD_PAUSE_MAX_MS from the hold's start; the
+		 * pause extends the deadline a full GR window past the last
+		 * quiet-hold pass, so the backend's retry/accept can finish
+		 * before the fallback would author. Announced once per hold.
+		 */
+		if (gsp_phys_gr_quiet_hold_active() &&
+		    time_before(jiffies, svc_fwd_hold.since +
+				msecs_to_jiffies(SVC_FWD_GR_HOLD_PAUSE_MAX_MS))) {
+			if (!svc_fwd_hold.pause_logged) {
+				svc_fwd_hold.pause_logged = true;
+				pr_warn("vgsp-gr: GR-HOLD-PAUSE va=0x%llx (backend wedge-quiet hold active; release authorship deferred under the bounded hold)\n",
+					svc_fwd_req.gr_push_va);
+			}
+			svc_fwd_hold.paused_until = jiffies +
+				msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS);
+			return false;
+		}
+		if (svc_fwd_hold.paused_until &&
+		    time_before(jiffies, svc_fwd_hold.paused_until))
+			return false;
 		return time_after(jiffies, svc_fwd_hold.since +
 				  msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS));
+	}
 	return time_after(jiffies, svc_fwd_hold.since +
 			  msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS));
 }
@@ -16324,6 +16378,9 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu,
 	svc_fwd_hold.relay_armed = svc_fwd_req.relay;
 	svc_fwd_hold.relay_index = svc_fwd_req.relay_index;
 	svc_fwd_hold.fault_delivered = false;
+	/* r420: a fresh hold starts with no WEDGE-QUIET deferral. */
+	svc_fwd_hold.paused_until = 0;
+	svc_fwd_hold.pause_logged = false;
 	svc.fwd_hold_engaged++;
 	svc.fwd_rel_pending = n;
 }
@@ -16350,6 +16407,8 @@ static void svc_fwd_hold_clear(void)
 	svc_fwd_hold.relay_armed = false;
 	svc_fwd_hold.relay_index = 0;
 	svc_fwd_hold.fault_delivered = false;
+	svc_fwd_hold.paused_until = 0;
+	svc_fwd_hold.pause_logged = false;
 	svc.fwd_rel_pending = 0;
 }
 
@@ -18063,6 +18122,27 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
 		ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms);
 		ch->gr_hold_exp_logged = false;
 	}
+	/*
+	 * r420: pause the per-launch consumption bound while the backend
+	 * deliberately holds the in-flight launch in its bounded WEDGE-QUIET
+	 * window. At the bound the pump falls through to the historical
+	 * consume, which authors the capture's QMD completion release in
+	 * software (GR-STALL-hold) - but the backend is still going to
+	 * submit/accept this very launch after the hold and copy its data
+	 * out; authoring now releases the client onto stale data (the r419
+	 * mid-boot escape). The quiet hold is a bounded msleep (the
+	 * backend's own caps), and the deferral here is bounded by
+	 * SVC_GR_QUIET_PAUSE_MAX_MS from the bound's arm time as
+	 * defence-in-depth. Rate-limited announcement.
+	 */
+	if (ch->gr_hold_dead_va && gsp_phys_gr_quiet_hold_active() &&
+	    time_before(jiffies, ch->gr_hold_dead_since +
+			msecs_to_jiffies(SVC_GR_QUIET_PAUSE_MAX_MS))) {
+		ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms);
+		pr_info_ratelimited("vgsp-gr: GR-HOLD-BOUND-PAUSE launch=0x%llx (backend wedge-quiet hold active; consumption bound deferred)\n",
+				    launch_va);
+		return false;
+	}
 	return time_after_eq(jiffies, ch->gr_hold_deadline);
 }
 
@@ -20779,7 +20859,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 										    svc_gr_launch_va(ch)) &&
 										   (ch->gr_hold_idx != (u32)idx ||
 										    time_before(jiffies,
-												ch->gr_hold_until))) {
+												ch->gr_hold_until) ||
+										    /*
+										     * r420: the backend's
+										     * bounded WEDGE-QUIET
+										     * hold outlives this
+										     * one-shot window; keep
+										     * re-holding (never
+										     * consume-with-sw-release)
+										     * while it is active, so
+										     * the launch is still
+										     * forwarded and copied out
+										     * after the hold instead of
+										     * being released onto stale
+										     * data.
+										     */
+										    gsp_phys_gr_quiet_hold_active())) {
 										u32 hold_ms = gsp_sema_gr_grace_ms ?
 											gsp_sema_gr_grace_ms : 8000;