vgsp: m178 GR launch reliability + diagnostics; tools: pin latching robustness

Kernel: authorship instrumentation + swrel gate for in-flight GR release VAs;
first-poll shortening + fast extended re-arm (params gr_retry_fast=1,
gr_first_poll_ms=2500); per-attempt win0 bank recompute + shadow recopy in the
outcome-retry loop; poll-break/deposit-readback diagnostics; C1-C10 sema
diagnostics (crel-entry, relx, opdef-clear, walk, pub-adv, hold-fallback,
busy-defer, CAND-OVERWRITE, inline recs); busy-defer hold instead of
software-release consume; counter fast path; QMD word 48/49 skip.
Tools: heartbeat thread, scanner-alive probe on reinstall, full periodic
re-assert, #nopid handling, per-client reset fix (debounce), dead-pid drop,
supervisor grace+threshold.

1 files changed, 333 insertions, 40 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index d21c5ea..f0f4baa 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -39,6 +39,7 @@
 #include <linux/seq_file.h>
 #include <linux/uaccess.h>
 #include <linux/jiffies.h>
+#include <linux/ktime.h>
 #include <linux/timekeeping.h>
 #include <linux/byteorder/generic.h>
 #include <linux/pci_regs.h>
@@ -1542,6 +1543,13 @@ enum gsp_phys_rmcall_buf {
  * past the existing hold budget.
  */
 #define GSP_PHYS_RMCALL_GR_RETRY_MS	2000u
+/*
+ * F2: the short re-poll after the first-timeout fast re-arm (extended
+ * re-arm + fresh ring entry). The engine ack for a recovered launch is
+ * observed within tens of ms (r198); 1500 ms covers the ctxsw path without
+ * burning another full window before the historical ladder continues.
+ */
+#define GSP_PHYS_RMCALL_GR_FAST_RETRY_MS 1500u
 #define GSP_PHYS_RMCALL_GR_ATTEMPTS	3u
 /* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */
 #define GSP_PHYS_RMCALL_GUEST_POLL_MS	200u
@@ -17160,6 +17168,142 @@ static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
 	dma_wmb();
 }
 
+/*
+ * F3 retry hygiene: an outcome retry re-publishes the same QMD. If the QMD's
+ * pointer pairs name the win0 shadow bank the previous attempt used, move the
+ * already-prepared param/program/bank pages into the alternate bank and
+ * rewrite the pairs (and the program prefetch) to the fresh VAs. The copies
+ * come from the module shadow pages - after relocation, inline replay and
+ * pointer fix-ups - so the engine-visible content is byte-identical to the
+ * first attempt and only the VA changes; the engine therefore cannot serve
+ * the retry from a cached fetch line of the previous attempt's bank. With
+ * gsp_phys_rmcall_gr_ptr_guest the pairs hold guest VAs, match nothing here,
+ * and the engine keeps reading the guest VAs exactly as before. Returns true
+ * when at least one pair moved (the caller must rebuild the method block).
+ */
+static bool gsp_phys_gr_win0_retry_rearm(u32 *qmd,
+					 const struct gsp_phys_gr_submit *req,
+					 u32 old_bank_base, u32 new_bank_base,
+					 u32 new_bank_limit)
+{
+	struct gsp_phys_boot_chunk *pool = &boot_stage.rmcall_gr_shadow_win0;
+	u64 old_lo = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+		     (u64)old_bank_base * GSP_PHYS_PAGE_SIZE;
+	u64 old_hi = old_lo +
+		     (u64)GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS *
+		     GSP_PHYS_PAGE_SIZE;
+	u32 next_slot = new_bank_base;
+	u32 i;
+	bool moved = false;
+
+	if (!pool->pages)
+		return false;
+	for (i = 0; i < req->n_pages; i++) {
+		const struct gsp_phys_gr_page *pg = &req->pages[i];
+		u32 w = pg->qmd_word;
+		u64 pair_va, new_va;
+		u32 off, old_slot;
+		u8 *src, *dst;
+
+		if (w != 32u && w != 48u &&
+		    !(w >= 34u && w <= 46u && !(w & 1u)))
+			continue;
+		pair_va = ((u64)(qmd[w + 1] & 0x1ffffu) << 32) | qmd[w];
+		if (pair_va < old_lo || pair_va >= old_hi)
+			continue;
+		if (next_slot >= new_bank_limit)
+			break;
+		off = (u32)(pair_va & (GSP_PHYS_PAGE_SIZE - 1));
+		old_slot = (u32)((pair_va -
+				  GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) /
+				 GSP_PHYS_PAGE_SIZE);
+		src = gsp_phys_chunk_page_va(pool, old_slot);
+		dst = gsp_phys_chunk_page_va(pool, next_slot);
+		if (!src || !dst)
+			break;
+		memcpy(dst, src, GSP_PHYS_PAGE_SIZE);
+		new_va = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+			 (u64)next_slot * GSP_PHYS_PAGE_SIZE + off;
+		qmd[w] = (u32)new_va;
+		qmd[w + 1] = (qmd[w + 1] & ~0x1ffffu) |
+			     ((u32)(new_va >> 32) & 0x1ffffu);
+		if (w == 48u) {
+			u64 pref = ((u64)(qmd[31] & 0x1ffu) << 40) |
+				   ((u64)qmd[8] << 8);
+
+			if ((pref & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) ==
+			    (pair_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) {
+				u64 np = new_va |
+					 (pref & (GSP_PHYS_PAGE_SIZE - 1));
+
+				qmd[8] = (u32)(np >> 8);
+				qmd[31] = (qmd[31] & ~0x1ffu) |
+					  ((u32)(np >> 40) & 0x1ffu);
+			}
+		}
+		next_slot++;
+		moved = true;
+	}
+	if (moved)
+		dma_wmb();
+	return moved;
+}
+
+/*
+ * F4-a: the first poll break, rendered from a noinline helper so its
+ * addresses/counter scratch never grow the (already warning-bound) submit
+ * frame. The USERD GP_GET/GP_PUT at the break discriminate an engine ack
+ * (GP_GET advanced, break latency long enough for a real retirement) from a
+ * host-authored word (break within a few ms of the doorbell, GP_GET frozen).
+ */
+static noinline void
+gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us,
+			   const struct gsp_phys_rmcall_ce_submit *sub)
+{
+	struct gsp_phys_boot_chunk *ud_c =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
+	u8 *ud = ud_c->pages ? gsp_phys_chunk_page_va(ud_c, 0) : NULL;
+	u32 usr_gpget = ud ?
+		get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() +
+				   GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0;
+
+	pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n",
+		w0, want_qmd, w1,
+		(u32)ktime_to_us(ktime_get()) - since_us,
+		usr_gpget, sub->gpget, sub->gpput);
+}
+
+/*
+ * F4-b: one-shot deposit readback at the first timeout (noinline for the same
+ * frame reason as above). Reads the qmdwin/shadow destination (or the guest
+ * QMD page) with the booter's existing unaligned host readback; w63 is the
+ * engine fetch sentinel.
+ */
+static noinline void
+gsp_phys_gr_deposit_rb_log(const struct gsp_phys_gr_submit *req,
+			   u8 *qmd_dst_mem)
+{
+	u8 *qp = qmd_dst_mem ? qmd_dst_mem :
+		(req->qmd_hpa ? (u8 *)__va(req->qmd_hpa) : NULL);
+
+	if (!qp)
+		return;
+	pr_info("vgsp-gr: deposit readback w0=0x%08x w4=0x%08x w12=0x%08x w13=0x%08x w18=0x%08x w19=0x%08x w23=0x%08x w32=0x%08x w33=0x%08x w48=0x%08x w49=0x%08x w63=0x%08x\n",
+		get_unaligned_le32(qp),
+		get_unaligned_le32(qp + 16),
+		get_unaligned_le32(qp + 48),
+		get_unaligned_le32(qp + 52),
+		get_unaligned_le32(qp + 72),
+		get_unaligned_le32(qp + 76),
+		get_unaligned_le32(qp + 92),
+		get_unaligned_le32(qp + 128),
+		get_unaligned_le32(qp + 132),
+		get_unaligned_le32(qp + 192),
+		get_unaligned_le32(qp + 196),
+		get_unaligned_le32(qp +
+				   GSP_PHYS_GR_QMD_SENTINEL_WORD * 4));
+}
+
 int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				     struct gsp_phys_backend_copy_result *out)
 {
@@ -17219,8 +17363,33 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	char qmdwin_note[32] = "";
 	bool rearmed = false;
 	u32 attempts = 0;
+	/*
+	 * F2/F4 state: the fast first-timeout re-arm runs once per launch; the
+	 * deposit readback is one-shot per launch (the first timeout is the
+	 * evidence window - the old r198 post-mortem only ran after attempt 3,
+	 * which recovered launches never reached); poll_ms is the window in
+	 * force for the current poll, so the timeout log reports it accurately.
+	 * Static like the other submit scratch: the single-inflight ordered
+	 * work item is the only caller, and statics keep the frame at its
+	 * warning-bound size.
+	 */
+	static bool fast_rearmed;
+	static bool deposit_rb_done;
+	static u16 poll_ms;
+	static u32 poll_start_us;
 	int ret = 0;
 
+	fast_rearmed = false;
+	deposit_rb_done = false;
+	/*
+	 * F2: the shortened first window is part of the fast path; with
+	 * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior
+	 * (the historical 5 s first window and ladder).
+	 */
+	poll_ms = gsp_phys_gr_retry_fast_value() ?
+		(u16)gsp_phys_gr_first_poll_ms_value() :
+		(u16)GSP_PHYS_RMCALL_GR_POLL_MS;
+
 	if (out)
 		memset(out, 0, sizeof(*out));
 	if (!req || !req->qmd ||
@@ -18184,14 +18353,15 @@ gr_page_unmapped:
 		gsp_phys_rmcall_gr_prelude_value(),
 		gsp_phys_rmcall_gr_qmdvirt_value(),
 		boot_stage.rmcall_token,
-		qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note);
+		qmd_rel, want_ch, poll_ms, qmdwin_note);
 	if (qmd_win_diag)
 		pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n",
 			qmd_dst, qmd_win_pde, qmd_win_installed,
 			(unsigned long long)boot_stage.rmcall_guest_pt_win_mask,
 			qmd_win_ret);
 
-	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS);
+	poll_start_us = (u32)ktime_to_us(ktime_get());
+	deadline = jiffies + msecs_to_jiffies(poll_ms);
 	for (;;) {
 		/*
 		 * m72: SKED transition recorder - sample the CWD/HWW registers
@@ -18241,17 +18411,81 @@ gr_page_unmapped:
 						ch_rel_off));
 		else
 			w1 = READ_ONCE(*(u32 *)(sem_va + 4));
-		if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch)
+		if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) {
+			/* F4-a: first poll break + USERD witnesses. */
+			gsp_phys_gr_poll_break_log(w0, want_qmd, w1,
+						   poll_start_us, &sub);
 			break;
+		}
 		if ((++grpoll_n % 100) == 0)
 			pr_info("vgsp-gr: poll +%ums qmd_rel=0x%08x ch_rel=0x%08x\n",
 				grpoll_n * GSP_PHYS_RMCALL_POLL_MS, w0, w1);
 		if (time_after_eq(jiffies, deadline)) {
 			gsp_phys_gr_timed_out++;
 			pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n",
-			       GSP_PHYS_RMCALL_GR_POLL_MS, w0, want_qmd, w1,
+			       poll_ms, w0, want_qmd, w1,
 			       want_ch, req->h_object, req->push_va);
 			/*
+			 * F4-b: one-shot deposit readback at the FIRST timeout
+			 * (the old post-mortem only ran after attempt 3, which
+			 * a recovered launch never reached). Reading the
+			 * qmdwin/shadow destination here proves whether THIS
+			 * attempt's inline-QMD copy landed and what the engine
+			 * would fetch (w63 is the fetch sentinel). Bound: one
+			 * line per launch.
+			 */
+			if (!deposit_rb_done) {
+				deposit_rb_done = true;
+				gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem);
+			}
+			/*
+			 * F2: fast re-arm at the first timeout. The m155
+			 * extended re-arm (lane-enable replay + GPFIFO/TSG
+			 * schedule + doorbell) plus a FRESH ring entry recovers
+			 * the r198 class in ~11 s only because it runs after two
+			 * full 5 s windows. Do it now and re-poll briefly; if it
+			 * still fails, fall through to the historical ladder
+			 * (the fast entry counts against the same attempt
+			 * budget, so the total number of host publishes and the
+			 * ring wrap exposure stay as before).
+			 */
+			if (gsp_phys_gr_retry_fast_value() && !fast_rearmed) {
+				int fr;
+
+				fast_rearmed = true;
+				attempts++;
+				mutex_lock(&gsp_phys_rm_lock);
+				fr = gsp_phys_rmcall_publish_locked(
+					bar0, boot_stage.rmcall_va_pb, words,
+					&sub);
+				if (!fr)
+					fr = gsp_phys_rmcall_gr_resched_locked(
+						bar0);
+				mutex_unlock(&gsp_phys_rm_lock);
+				if (!fr) {
+					gsp_phys_gr_retries++;
+					poll_ms =
+						GSP_PHYS_RMCALL_GR_FAST_RETRY_MS;
+					pr_info("vgsp-gr: fast re-arm after first timeout: extended re-arm + fresh ring entry slot=%u published; re-polling %u ms (attempts %u/%u)\n",
+						sub.slot, poll_ms, attempts,
+						GSP_PHYS_RMCALL_GR_ATTEMPTS);
+					deadline = jiffies +
+						msecs_to_jiffies(poll_ms);
+					/*
+					 * The immediate m142 re-arm would
+					 * only duplicate the resched that just
+					 * ran; the ladder below still owns the
+					 * warm-sem gate and the remaining
+					 * fresh entries.
+					 */
+					rearmed = true;
+					grpoll_n = 0;
+					continue;
+				}
+				pr_info("vgsp-gr: fast re-arm failed rc=%d; falling through to the timeout ladder\n",
+					fr);
+			}
+			/*
 			 * m142: one bounded re-arm before declaring the launch
 			 * dead. Re-issue the stage-4 schedule controls and re-ring
 			 * the doorbell, then poll again; if the channel was merely
@@ -18266,11 +18500,11 @@ gr_page_unmapped:
 				rr = gsp_phys_rmcall_gr_resched_locked(bar0);
 				mutex_unlock(&gsp_phys_rm_lock);
 				if (!rr) {
+					poll_ms = GSP_PHYS_RMCALL_GR_POLL_MS;
 					pr_info("vgsp-gr: rearm after timeout: schedule re-issued + doorbell re-rung; re-polling %u ms\n",
-						GSP_PHYS_RMCALL_GR_POLL_MS);
+						poll_ms);
 					deadline = jiffies +
-						msecs_to_jiffies(
-						 GSP_PHYS_RMCALL_GR_POLL_MS);
+						msecs_to_jiffies(poll_ms);
 					grpoll_n = 0;
 					continue;
 				}
@@ -18327,14 +18561,13 @@ gr_page_unmapped:
 				mutex_unlock(&gsp_phys_rm_lock);
 				if (!pr_ret) {
 					gsp_phys_gr_retries++;
+					poll_ms = GSP_PHYS_RMCALL_GR_RETRY_MS;
 					pr_info("vgsp-gr: retry %u/%u: fresh ring entry slot=%u published + rescheduled; re-polling %u ms\n",
 						attempts,
 						GSP_PHYS_RMCALL_GR_ATTEMPTS,
-						sub.slot,
-						GSP_PHYS_RMCALL_GR_RETRY_MS);
+						sub.slot, poll_ms);
 					deadline = jiffies +
-						msecs_to_jiffies(
-						 GSP_PHYS_RMCALL_GR_RETRY_MS);
+						msecs_to_jiffies(poll_ms);
 					rearmed = false;
 					grpoll_n = 0;
 					continue;
@@ -18737,37 +18970,86 @@ gr_page_unmapped:
 				WRITE_ONCE(*(u32 *)sem_va, 0);
 				WRITE_ONCE(*(u32 *)(sem_va + 4), 0);
 				wmb();
-				if (qmdwin_on &&
-				    boot_stage.rmcall_gr_qmdwin.pages) {
-					epoch = ++boot_stage.rmcall_gr_submit_epoch;
-					if (!epoch)
+				/*
+				 * F3 retry hygiene: recompute the win0 bank
+				 * and slot limit for THIS attempt (they were
+				 * fixed at the first submission's epoch) and
+				 * re-copy the prepared param/program/bank
+				 * shadow pages into the alternate bank before
+				 * the re-publish. The qmdwin bank swap (below)
+				 * is unchanged; when both are active they share
+				 * the same fresh epoch.
+				 */
+				{
+					bool win0_on =
+						gsp_phys_rmcall_gr_shadow_win0_value() &&
+						boot_stage.rmcall_gr_shadow_win0.pages;
+					bool win0_moved = false;
+					bool qmdwin_swap = qmdwin_on &&
+						boot_stage.rmcall_gr_qmdwin.pages;
+
+					if (qmdwin_swap || win0_on) {
 						epoch = ++boot_stage.rmcall_gr_submit_epoch;
-					qw_bank = (epoch & 1u) %
-						GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS;
-					qmd_dst = boot_stage.rmcall_gr_qmdwin_va +
-						(u64)qw_bank *
-						GSP_PHYS_PAGE_SIZE;
-					qmd_dst_mem = gsp_phys_chunk_page_va(
-						&boot_stage.rmcall_gr_qmdwin,
-						qw_bank);
-					if (qmd_dst_mem) {
-						memcpy(qmd_dst_mem, qmd,
-						       GSP_PHYS_GR_QMD_WORDS * 4);
-						put_unaligned_le32(
-							GSP_PHYS_GR_QMD_SENTINEL,
-							qmd_dst_mem +
-							GSP_PHYS_GR_QMD_SENTINEL_WORD * 4);
+						if (!epoch)
+							epoch = ++boot_stage.rmcall_gr_submit_epoch;
+						boot_stage.rmcall_gr_shadow_win0_used =
+							(epoch & 1u) *
+							GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS;
+						if (win0_on)
+							win0_moved =
+								gsp_phys_gr_win0_retry_rearm(
+									qmd, req,
+									win0_bank_base,
+									boot_stage.rmcall_gr_shadow_win0_used,
+									boot_stage.rmcall_gr_shadow_win0_used +
+									GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS);
+						win0_bank_base =
+							boot_stage.rmcall_gr_shadow_win0_used;
+						shadow_slot_limit =
+							win0_bank_base +
+							GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS;
 					}
-					words = gsp_phys_rmcall_build_gr_launch(
-						methods, req, qmd, qmd_dst,
-						ch_sem_va, want_ch);
-					memcpy(gsp_phys_chunk_page_va(pb, 0),
-					       methods, words * 4);
-					pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + PB rebuilt\n",
-						attempts, qw_bank, qmd_dst);
-				} else {
-					pr_info("vgsp-gr: outcome retry %u: release words zeroed (qmdwin off)\n",
-						attempts);
+					if (qmdwin_swap) {
+						qw_bank = (epoch & 1u) %
+							GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS;
+						qmd_dst =
+							boot_stage.rmcall_gr_qmdwin_va +
+							(u64)qw_bank *
+							GSP_PHYS_PAGE_SIZE;
+						qmd_dst_mem =
+							gsp_phys_chunk_page_va(
+								&boot_stage.rmcall_gr_qmdwin,
+								qw_bank);
+						if (qmd_dst_mem) {
+							memcpy(qmd_dst_mem, qmd,
+							       GSP_PHYS_GR_QMD_WORDS * 4);
+							put_unaligned_le32(
+								GSP_PHYS_GR_QMD_SENTINEL,
+								qmd_dst_mem +
+								GSP_PHYS_GR_QMD_SENTINEL_WORD * 4);
+						}
+					}
+					if (qmdwin_swap || win0_moved) {
+						words = gsp_phys_rmcall_build_gr_launch(
+							methods, req, qmd,
+							qmd_dst, ch_sem_va,
+							want_ch);
+						memcpy(gsp_phys_chunk_page_va(pb, 0),
+						       methods, words * 4);
+					}
+					if (qmdwin_swap)
+						pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n",
+							attempts, qw_bank,
+							qmd_dst, win0_bank_base,
+							win0_moved ?
+							 " (shadow pages re-copied)" :
+							 "");
+					else if (win0_moved)
+						pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh win0 bank %u (shadow pages re-copied) + PB rebuilt\n",
+							attempts, win0_bank_base);
+					else
+						pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n",
+							attempts);
 				}
 				gsp_phys_gr_fill_bounce(req, bounce_csum_in);
 				mutex_lock(&gsp_phys_rm_lock);
@@ -18871,6 +19153,17 @@ gr_page_unmapped:
 			if (wrote == 0) {
 				bool restored = false;
 
+				/*
+				 * C6 (GAP-2 A1): the no-write accept. Unconditional
+				 * by design - it is the rare, load-bearing event
+				 * (zero engine writes still accepted as success),
+				 * so it must be visible even when no release word
+				 * needed restoring.
+				 */
+				pr_info("vgsp-gr: NOWRITE h=0x%x push=0x%llx n_data=%u wrote=0 qmd_rel=0x%x\n",
+					req->h_object, req->push_va, req->n_data,
+					want_qmd);
+
 				if (qmd_rel_hpa) {
 					u8 *rp = (u8 *)__va(qmd_rel_hpa) +
 						 qmd_rel_off;