merge ws/native-discovery: kernel-native CE+GR guest-truth discovery (FORENSICS-v2 H1-H3, ISSUE-0001)

1 files changed, 817 insertions, 72 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 30b349b..ddc7078 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -389,6 +389,19 @@ static u32 svc_pb_n;
 static u32 svc_ring_n;
 static u8 svc_pb_passes;
 static u64 svc_pb_last;
+/*
+ * Pass budget for svc_pb_discover: stop after SVC_PB_IDLE_PASSES
+ * consecutive passes with no new base (the discovery has drained), with a
+ * hard cap so a channel that has no push-buffer base page cannot keep the
+ * pass running for the boot. The old code stopped at a flat 12 passes,
+ * which could retire discovery before a later channel/session's base
+ * appeared (ISSUE-0001 durable direction).
+ */
+#define SVC_PB_IDLE_PASSES	6
+#define SVC_PB_PASS_MAX		48
+static u32 svc_pb_idle;		/* consecutive passes with no new base */
+static u32 svc_pb_last_n;	/* bases before the active pass */
+static unsigned long svc_pb_done;	/* CE ranks whose push buffer is pinned */
 
 static bool svc_pb_first_push(const u8 *buf)
 {
@@ -516,6 +529,8 @@ struct sema_svc {
 	u64			gpfifo_skipped;		/* stale entries consumed */
 	u64			gpfifo_rescued;		/* unreadable push made readable
 						 * by the content rescue */
+	u64			resc_run;		/* pushes pinned by the multi-page
+						 * content rescue run */
 	u64			resc_resync;		/* expectation re-synced down to
 						 * the content watermark */
 	u64			releases_seen;
@@ -1298,7 +1313,23 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va,
 static u64 svc_pd_base;
 static int svc_pd_ap;
 static u32 svc_pd_entries;
-static u8  svc_pd_tried;
+/*
+ * Discovery is retryable, not one-shot (ISSUE-0001 durable direction).
+ *
+ * The first call can happen before the client has written the page tables
+ * (the sva/spa samples do not exist yet) or before enough verified pairs are
+ * known; the old latch (`svc_pd_tried = 1` before any scan) then disabled the
+ * only kernel-native source that covers pool/push windows the content-scanned
+ * map and the published directories miss, for the rest of the boot, even
+ * though the samples it needs appear seconds later. Re-arm on new evidence
+ * instead: an attempt that could not run does not latch, and a scan runs only
+ * when the sample set changed since the last one (fingerprint), bounded by
+ * SVC_PD_ATTEMPTS_MAX scans per boot - the same order as the old
+ * one-per-pairing re-arm.
+ */
+#define SVC_PD_ATTEMPTS_MAX	8
+static u8  svc_pd_attempts;	/* scans actually run (not sample misses) */
+static u64 svc_pd_ev_hash;	/* fingerprint of the sample set last scanned */
 static u8  svc_pd_found;
 #define SVC_PD_SAMPLES	8
 #define SVC_PD_MIN_HITS	4
@@ -1343,9 +1374,8 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
 	u8 *buf;
 	int pass;
 
-	if (svc_pd_tried)
+	if (svc_pd_found || svc_pd_attempts >= SVC_PD_ATTEMPTS_MAX)
 		return;
-	svc_pd_tried = 1;
 
 	/*
 	 * Pinned push-buffer pairs are the strongest samples available: they
@@ -1365,7 +1395,7 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
 	if (n < SVC_PD_MIN_HITS) {
 		n = 0;
 		if (svc.map_n < SVC_PD_SAMPLES)
-			return;
+			return;	/* no evidence yet: retry when pairs appear */
 		step = svc.map_n / SVC_PD_SAMPLES;
 		for (i = 0; i < SVC_PD_SAMPLES; i++) {
 			sva[i] = svc.map[i * step].gpu_page;
@@ -1377,6 +1407,24 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
 	}
 	if (!n)
 		return;
+	/*
+	 * One scan per distinct sample set: a miss on the same evidence cannot
+	 * succeed on a retry, and the scan costs a full RAM walk. A changed
+	 * sample set (a new pinned pair, or a rebuilt map) is new evidence and
+	 * re-arms the scan. The fingerprint uses the values, not just the
+	 * count, because reaching four pinned pairs replaces the map samples
+	 * one-for-one and can leave the count unchanged.
+	 */
+	{
+		u64 ev = n;
+
+		for (i = 0; i < n; i++)
+			ev = (ev + sva[i] + (spa[i] << 12)) * 1000003;
+		if (svc_pd_attempts && ev == svc_pd_ev_hash)
+			return;
+		svc_pd_ev_hash = ev;
+	}
+	svc_pd_attempts++;
 
 	/* Guest RAM first, then the emulated vidmem. Page directories are page
 	 * aligned, and every VA here is below 2^47, so the root entry that
@@ -1970,7 +2018,14 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 		svc_lat_store(va, *addr, *ap);
 		return 0;
 	}
-	if (!svc_pd_tried) {
+	if (!svc_pd_found) {
+		/*
+		 * Retryable discovery: on a boot where the directory is found
+		 * only after the client's page tables appear, a later call
+		 * with fresh verified pairs re-runs the scan. svc_pd_discover
+		 * self-gates (evidence, attempt cap), so this is cheap when
+		 * nothing changed.
+		 */
 		svc_pd_discover(vgpu);
 		if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
 			*addr += va & 0xfff;
@@ -2863,6 +2918,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n",
 		svc.counter_pub_calls, svc.gr_rel_applied,
 		svc.sem_write_reresolve, svc.pushpg_fallback);
+	if (svc.resc_run)
+		pr_info("sema-svc:   rescue_run=%llu (multi-page content pins)\n",
+			svc.resc_run);
 	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n",
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
 		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
@@ -4645,13 +4703,50 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
 
 	if (ch->engine_type || n_gr <= 0)
 		return;
+	if (ch->gr_index >= n_gr)
+		return;
+	va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL;
 	if (ch->counter_pa) {
+		/*
+		 * FORENSICS-v2 anomaly 6: a slot derived in an earlier pass
+		 * can disagree with a later-arriving injected pin (r1:
+		 * derived 0x123a43ff0 vs pin-backed 0x123a28ff0, off by
+		 * 0x1b000). Injected pins are the authoritative guest truth -
+		 * svc_counter_learn re-checks them when a release decodes -
+		 * but nothing re-checked them while counter_pa was already
+		 * registered and no push had been decoded yet, so a publish
+		 * would have advanced a page the client never polls. Re-check
+		 * the rank's slot on every pass and re-register on a mismatch
+		 * so the publish is proof-gated through the pin.
+		 */
+		u64 ipa = 0;
+		int iap = SVC_APERTURE_SYS;
+
+		if (!svc_ipin_lookup(va, &ipa, &iap) &&
+		    iap == SVC_APERTURE_SYS &&
+		    (ipa & ~0x3ULL) != ch->counter_pa) {
+			unsigned long flags;
+			u64 old = ch->counter_pa;
+			int j;
+
+			spin_lock_irqsave(&svc.chan_lock, flags);
+			ch->counter_pa = ipa & ~0x3ULL;
+			for (j = 0; j < SVC_MAX_CHANNELS; j++) {
+				if (svc.chan[j].valid &&
+				    svc.chan[j].h_object == ch->h_object) {
+					svc.chan[j].counter_pa =
+						ch->counter_pa;
+					break;
+				}
+			}
+			spin_unlock_irqrestore(&svc.chan_lock, flags);
+			pr_info("sema-chan: counter pin override h=0x%x rank=%u pa=0x%llx (was 0x%llx; injected pin is authoritative)\n",
+				ch->h_object, ch->gr_index,
+				ch->counter_pa, old);
+		}
 		svc_counter_flush_pending(vgpu, ch);
 		return;
 	}
-	if (ch->gr_index >= n_gr)
-		return;
-	va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL;
 	/*
 	 * The injected-pin table is the authoritative source; only it may
 	 * shadow a self-discovered map entry (which can hold a stale
@@ -6494,30 +6589,67 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu)
 	unsigned long flags;
 	u8 *buf;
 	u64 gpa;
-	bool ce = false;
+	bool ce = false, need = false;
 	int i;
 
-	if (svc_pb_passes >= 12 || svc_pb_n >= SVC_PB_MAX)
-		return;
-	if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ))
-		return;
-
 	spin_lock_irqsave(&svc.chan_lock, flags);
-	for (i = 0; i < SVC_MAX_CHANNELS; i++)
-		if (svc.chan[i].valid && svc.chan[i].engine_type) {
-			ce = true;
-			break;
-		}
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		u32 slot, k;
+		bool have = false;
+
+		if (!svc.chan[i].valid || !svc.chan[i].engine_type)
+			continue;
+		ce = true;
+		if (svc.chan[i].ce_index >= 32)
+			continue;
+		if (test_bit(svc.chan[i].ce_index, &svc_pb_done))
+			continue;
+		/*
+		 * "Need" means no content-discovered base exists for this
+		 * channel's release slot yet. Once the base is recorded the
+		 * pass can idle out even if the pump has not paired the
+		 * window; pairing is a pump concern, not a discovery one.
+		 */
+		slot = 4 * svc.chan[i].ce_index;
+		for (k = 0; k < svc_pb_n; k++)
+			if (svc_pb[k].slot == slot) {
+				have = true;
+				break;
+			}
+		if (!have)
+			need = true;
+	}
 	spin_unlock_irqrestore(&svc.chan_lock, flags);
 	if (!ce)
 		return;
 
+	/*
+	 * Discovery stays alive while it is still finding bases or while a
+	 * registered CE channel has no paired base (ISSUE-0001 durable
+	 * direction). The old flat 12-pass budget stopped the pass even when
+	 * a channel had appeared (or reappeared with a new client session)
+	 * after the budget was spent, so its push-buffer base was never
+	 * paired and every push depended on the slower per-page rescue and
+	 * watcher pins. Stop only after SVC_PB_IDLE_PASSES consecutive
+	 * passes with no new base AND no unpaired channel, with a hard cap
+	 * so a channel that genuinely has no base page cannot scan forever.
+	 * The 3 s rate limit and the chunked walk keep the service thread
+	 * responsive.
+	 */
+	if (svc_pb_n >= SVC_PB_MAX || svc_pb_passes >= SVC_PB_PASS_MAX)
+		return;
+	if (svc_pb_idle >= SVC_PB_IDLE_PASSES && !need)
+		return;
+	if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ))
+		return;
+
 	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
 	if (!buf)
 		return;
 
 	svc_pb_passes++;
 	svc_pb_last = jiffies;
+	svc_pb_last_n = svc_pb_n;
 	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
 		u32 o;
 
@@ -6527,8 +6659,12 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu)
 			svc_pb_note(vgpu, buf + o, gpa + o);
 	}
 	kfree(buf);
-	pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u)\n",
-		svc_pb_passes, svc_pb_n, svc_ring_n);
+	if (svc_pb_n > svc_pb_last_n)
+		svc_pb_idle = 0;
+	else
+		svc_pb_idle++;
+	pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u, idle %u)\n",
+		svc_pb_passes, svc_pb_n, svc_ring_n, svc_pb_idle);
 }
 
 /*
@@ -6975,29 +7111,48 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as,
  * Whichever candidate reads a coherent get/put pair wins and is cached. An
  * idle candidate (0/0) is not accepted until it shows activity, so a zeroed
  * page is never mistaken for the control block.
+ *
+ * H2 (FORENSICS-v2): every ring-derived and registered candidate is tried in
+ * BOTH apertures. The registered `userd_as` can be mis-classified - v2's GR
+ * channel read its USERD through FB and got 0/0 (`probe rc=-11`, -EAGAIN)
+ * while the same boot pins the GR pushbuffer in guest sysmem - and the old
+ * FB-only candidate list then had no way to reach the real block. The first
+ * aperture stays the channel's own; a wrong second aperture only costs a
+ * failed probe (the values must still read as a coherent put/get pair).
  */
 static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			  u64 ring_pa, int ring_ap, u32 *put, u32 *get)
 {
-	u64 cand[4];
-	u32 as[4], nc = 0, i;
+	u64 cand[8];
+	u32 as[8], nc = 0, i;
+	int ring_alt = ring_ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS
+						  : SVC_APERTURE_FB;
 
-	if (ring_ap == SVC_APERTURE_FB && ring_pa) {
-		cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB;
-		cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB;
+	if (ring_pa) {
+		cand[nc] = ring_pa + 0x2000; as[nc++] = ring_ap;
+		cand[nc] = ring_pa + 0x2000; as[nc++] = ring_alt;
+		cand[nc] = ring_pa + 0x1000; as[nc++] = ring_ap;
+		cand[nc] = ring_pa + 0x1000; as[nc++] = ring_alt;
 		/*
-		 * UVM's channel allocations place the USERD one 0x10000 block
-		 * after the ring (observed for all eight CE channels: ring at
-		 * FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). The
-		 * allocation parameters' userdMem base is relative to a client
-		 * pool and does not describe this layout.
+		 * UVM's CE channel allocations place the USERD one 0x10000
+		 * block after the ring (observed for all eight CE channels:
+		 * ring at FB 0x80000 + 0x20000*i, USERD at ring + 0x10000).
+		 * The allocation parameters' userdMem base is relative to a
+		 * client pool and does not describe this layout.
 		 */
 		if (ch->engine_type != 0) {
 			cand[nc] = ring_pa + 0x10000;
-			as[nc++] = SVC_APERTURE_FB;
+			as[nc++] = ring_ap;
+			cand[nc] = ring_pa + 0x10000;
+			as[nc++] = ring_alt;
 		}
 	}
-	cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as;
+	if (ch->userd_pa) {
+		cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as;
+		cand[nc] = ch->userd_pa;
+		as[nc++] = ch->userd_as == SVC_APERTURE_FB
+				   ? SVC_APERTURE_SYS : SVC_APERTURE_FB;
+	}
 
 	for (i = 0; i < nc; i++) {
 		if (!cand[i])
@@ -7168,6 +7323,356 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 bool gsp_sema_gr_rings = true;
 
 /*
+ * Validate a candidate translation for a GR ring: the first GPFIFO entry
+ * must decode to a 1 MB aligned push in the client VA window with a sane
+ * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the
+ * same translation. Returns false (candidate rejected) on any unreadable
+ * or implausible content, so a wrong translation costs a failed probe, not
+ * a wrong pin. `put`/`get` are set to zero for a readable idle pair.
+ */
+static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu,
+				  const struct svc_channel *ch, u64 cand_pa,
+				  int cand_ap, u64 *va, u32 *len, u32 *put,
+				  u32 *get)
+{
+	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	u32 e0, e1;
+	int prc;
+
+	if (svc_read_ap(vgpu, cand_pa, cand_ap, e, sizeof(e)))
+		return false;
+	memcpy(&e0, e, 4);
+	memcpy(&e1, e + 4, 4);
+	*va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+	*len = ((e1 >> 10) & 0x1fffff) << 2;
+	if ((*va & 0xfffff) || *va < SEMA_VA_LO || *va >= SEMA_VA_HI ||
+	    *len < 4 || *len > 0x10000)
+		return false;
+	/*
+	 * Gate: the USERD must be readable. rc==0 is an active get/put
+	 * pair; rc==-EAGAIN is a readable idle (0/0) block, accepted so a
+	 * freshly created channel is not rejected. -EINVAL (implausible
+	 * values) and -ENOENT (unreadable) reject the candidate.
+	 */
+	*put = 0;
+	*get = 0;
+	prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap,
+			      ch->gpfifo_entries, put, get);
+	return prc == 0 || prc == -EAGAIN;
+}
+
+/*
+ * H1 last resort - kernel-native content discovery of the GR ring/USERD.
+ *
+ * FORENSICS-v2 H1: the guest authors its eight GR context-init pushes
+ * (release records `[200426c0][00000002][0400ffXX][00000001]`, verified
+ * byte-for-byte in guest RAM; m104 geometry) but the host resolves neither
+ * the ring VA (0x200200000 + 0x3000*rank) nor its USERD, so
+ * svc_pump_channels never reads a GR GPFIFO entry: ring_auto=6 (CE only),
+ * pub_calls=0, gr_rel=0, work_gr req=0, and the client spins at DEVICE.
+ *
+ * The deterministic sources are tried first (per channel: the USERD-derived
+ * candidates in both apertures, the FB convention, an existing pin, and
+ * svc_lookup_trusted - the driver-published VAS walk that resolved the CE
+ * pool page `src=trusted` in v2, 11-hostdmesg-v2-r1.log:2313). This pass is
+ * the bounded last resort when none of them knows the ring VA: locate the
+ * ring page and the release-bearing push page by exact content in guest RAM.
+ *
+ * Content truth (m104 `17-host-m104-r1.txt:169`, handoff ~2576-2597):
+ *   - rank r (0..7) owns counter slot VA 0x20400fff0 - 0x10*r, and its
+ *     ctx-init push carries `[200426c0][00000002][0400ffXX][00000001]`
+ *     with the group at push offset len-20, the push page-aligned at VA
+ *     0x200400000 + 0x400000*r (ranks 0..7 observed; the PA backing
+ *     differs per boot);
+ *   - the ring chunk is VA 0x200200000 + 0x3000*r (the registered
+ *     gpfifo_va), entry0 at page offset 0 pointing at that push VA with
+ *     the same length; the USERD follows at +0x2000.
+ *
+ * One 64 KB-chunked pass over the registered guest RAM records up to
+ * SVC_GR_CONTENT_CANDS ring-entry and release-group candidates per rank,
+ * then accepts a (ring, push) pair only when the whole gate passes: entry0
+ * bytes exact (push VA + length), the push page parses start to end as a
+ * method stream, the release group sits exactly at len-20 with this rank's
+ * slot and payload 1, and the USERD reads back a plausible get/put pair
+ * (or a readable idle 0/0). Only then are the ring VA and the push VA page
+ * pinned (both SYS); a wrong or stale candidate pins nothing. Bounded: one
+ * pass per 2 s while a registered rank 0..7 has no translation, and
+ * SVC_GR_CONTENT_PASS_MAX passes per boot.
+ */
+#define SVC_GR_CONTENT_RANKS	8
+#define SVC_GR_PUSH_VA_BASE	0x200400000ULL
+#define SVC_GR_PUSH_VA_STRIDE	0x400000ULL
+#define SVC_GR_RING_VA_BASE	0x200200000ULL
+#define SVC_GR_SLOT_LO_BASE	0x0400fff0ULL
+#define SVC_GR_SLOT_LO_STRIDE	0x10ULL
+#define SVC_GR_REL_HDR		0x200426c0ULL
+#define SVC_GR_REL_HI		2ULL
+#define SVC_GR_CONTENT_CANDS	4
+#define SVC_GR_CONTENT_PASS_MAX	60
+
+struct svc_gr_content_cand {
+	u64	pa_page;
+	u32	len;
+};
+
+static struct svc_gr_content_cand
+	svc_gr_content_ring[SVC_GR_CONTENT_RANKS][SVC_GR_CONTENT_CANDS];
+static struct svc_gr_content_cand
+	svc_gr_content_push[SVC_GR_CONTENT_RANKS][SVC_GR_CONTENT_CANDS];
+static u8 svc_gr_content_nring[SVC_GR_CONTENT_RANKS];
+static u8 svc_gr_content_npush[SVC_GR_CONTENT_RANKS];
+static unsigned long svc_gr_content_last;
+static u32 svc_gr_content_passes;
+static bool svc_gr_content_cap_log;
+
+/*
+ * A ring page's bytes after entry0 are either zero (a fresh ring with
+ * put==1) or further plausible GPFIFO entries; a pageful of unrelated data
+ * rejects the candidate.
+ */
+static bool svc_gr_ring_tail_ok(const u8 *p, u32 n)
+{
+	u32 i;
+
+	for (i = 0; i + 8 <= n; i += 8) {
+		u32 e0, e1;
+		u64 va;
+		u32 len;
+
+		memcpy(&e0, p + i, 4);
+		memcpy(&e1, p + i + 4, 4);
+		if (!e0 && !e1)
+			continue;
+		va = ((u64)(e1 & 0xff) << 32) | (e0 & 0xfffffffc);
+		len = ((e1 >> 10) & 0x1fffff) << 2;
+		if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
+		    len < 4 || len > 0x10000)
+			return false;
+	}
+	return true;
+}
+
+static void svc_gr_content_record(struct svc_gr_content_cand *tab, u8 *n,
+				  u64 pa_page, u32 len)
+{
+	u32 i;
+
+	for (i = 0; i < *n; i++)
+		if (tab[i].pa_page == pa_page && tab[i].len == len)
+			return;
+	if (*n >= SVC_GR_CONTENT_CANDS)
+		return;
+	tab[*n].pa_page = pa_page;
+	tab[*n].len = len;
+	(*n)++;
+}
+
+/*
+ * Validate one (ring, push) pair from content and install both pins.
+ * Everything is read back from guest RAM: the ring's entry0 must be the
+ * exact (push VA, len) pair, the push page must parse as a complete method
+ * stream, the release group must sit exactly at len-20 with this rank's
+ * slot and payload 1, and the USERD must be readable. Returns true when
+ * the pins were installed.
+ */
+static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank,
+			       u64 ring_va, u32 entries, u64 ring_pa,
+			       u64 push_pa, u32 len)
+{
+	u64 push_va = SVC_GR_PUSH_VA_BASE + SVC_GR_PUSH_VA_STRIDE * rank;
+	u32 slot_lo = (u32)(SVC_GR_SLOT_LO_BASE -
+			    SVC_GR_SLOT_LO_STRIDE * rank);
+	u8 e[SVC_GPFIFO_ENTRY_SIZE], tail[20];
+	u8 *full;
+	u32 e0, e1, put = 0, get = 0, hdr, hi, lo, pay;
+	u64 va;
+	int prc;
+
+	if (len < 24 || svc_read(vgpu, ring_pa, e, sizeof(e)))
+		return false;
+	memcpy(&e0, e, 4);
+	memcpy(&e1, e + 4, 4);
+	va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+	if (va != push_va || (((e1 >> 10) & 0x1fffff) << 2) != len)
+		return false;
+	full = kmalloc(len, GFP_KERNEL);
+	if (!full)
+		return false;
+	if (svc_read(vgpu, push_pa, full, len) ||
+	    !svc_stream_parse(full, len)) {
+		kfree(full);
+		return false;
+	}
+	kfree(full);
+	if (svc_read(vgpu, push_pa + len - 20, tail, sizeof(tail)))
+		return false;
+	memcpy(&hdr, tail + 0, 4);
+	memcpy(&hi, tail + 4, 4);
+	memcpy(&lo, tail + 8, 4);
+	memcpy(&pay, tail + 12, 4);
+	if (hdr != SVC_GR_REL_HDR || hi != SVC_GR_REL_HI ||
+	    lo != slot_lo || pay != 1)
+		return false;
+	prc = svc_userd_probe(vgpu, ring_pa + SVC_GR_USERD_OFF,
+			      SVC_APERTURE_SYS, entries, &put, &get);
+	if (prc && prc != -EAGAIN)
+		return false;
+	svc_pin_store(ring_va, ring_pa, SVC_APERTURE_SYS);
+	svc_lat_store(ring_va, ring_pa, SVC_APERTURE_SYS);
+	svc_mark_hot(ring_pa);
+	svc_pin_store(push_va, push_pa, SVC_APERTURE_SYS);
+	svc_lat_store(push_va, push_pa, SVC_APERTURE_SYS);
+	svc_mark_hot(push_pa);
+	pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n",
+		rank, ring_va, ring_pa, push_va, len, push_pa, prc, put, get);
+	return true;
+}
+
+static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
+{
+	u64 gva[SVC_GR_CONTENT_RANKS] = { 0 };
+	u32 ent[SVC_GR_CONTENT_RANKS] = { 0 };
+	unsigned long flags;
+	u8 *buf;
+	bool need = false;
+	u64 gpa;
+	u32 i, o;
+
+	if (!gsp_sema_gr_rings)
+		return;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		struct svc_channel *c = &svc.chan[i];
+		u64 ppa = 0;
+		int pap = 0;
+		u32 r = c->gr_index;
+
+		if (!c->valid || c->engine_type || r >= SVC_GR_CONTENT_RANKS)
+			continue;
+		if (test_bit(r, &svc_gr_ring_done))
+			continue;
+		if (c->gpfifo_va !=
+		    SVC_GR_RING_VA_BASE + SVC_GR_RING_STRIDE * r)
+			continue;	/* layout changed: stay on the safe paths */
+		gva[r] = c->gpfifo_va;
+		ent[r] = c->gpfifo_entries;
+		if (!svc_pin_lookup_any(c->gpfifo_va, &ppa, &pap))
+			continue;	/* translated: the autopin will latch it */
+		need = true;
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+	if (!need)
+		return;
+
+	if (svc_gr_content_passes >= SVC_GR_CONTENT_PASS_MAX) {
+		if (!svc_gr_content_cap_log) {
+			svc_gr_content_cap_log = true;
+			pr_info("sema-chan: GR content discovery exhausted (%u passes, no validated ring)\n",
+				svc_gr_content_passes);
+		}
+		return;
+	}
+	if (time_before(jiffies, svc_gr_content_last + 2 * HZ))
+		return;
+	svc_gr_content_last = jiffies;
+	svc_gr_content_passes++;
+	memset(svc_gr_content_ring, 0, sizeof(svc_gr_content_ring));
+	memset(svc_gr_content_push, 0, sizeof(svc_gr_content_push));
+	memset(svc_gr_content_nring, 0, sizeof(svc_gr_content_nring));
+	memset(svc_gr_content_npush, 0, sizeof(svc_gr_content_npush));
+
+	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) {
+			u32 w0;
+
+			memcpy(&w0, buf + o, 4);
+			if (w0 == (u32)SVC_GR_REL_HDR) {
+				u32 hi, lo, pay, r, oip, len;
+
+				memcpy(&hi, buf + o + 4, 4);
+				memcpy(&lo, buf + o + 8, 4);
+				memcpy(&pay, buf + o + 12, 4);
+				if (hi != SVC_GR_REL_HI || pay != 1)
+					continue;
+				if ((u64)lo > SVC_GR_SLOT_LO_BASE ||
+				    (SVC_GR_SLOT_LO_BASE - lo) %
+					    SVC_GR_SLOT_LO_STRIDE)
+					continue;
+				r = (u32)((SVC_GR_SLOT_LO_BASE - lo) /
+					  SVC_GR_SLOT_LO_STRIDE);
+				if (r >= SVC_GR_CONTENT_RANKS || !gva[r])
+					continue;
+				oip = (u32)((gpa + o) &
+					    (SEMA_PAGE_SIZE - 1));
+				if (oip + 20 > SEMA_PAGE_SIZE)
+					continue;
+				/* page-aligned push start: len = oip + group */
+				len = oip + 20;
+				svc_gr_content_record(
+					svc_gr_content_push[r],
+					&svc_gr_content_npush[r],
+					(gpa + o) &
+						~(u64)(SEMA_PAGE_SIZE - 1),
+					len);
+			} else if ((w0 & 0x3fffff) == 0 &&
+				   w0 >= 0x00400000 && w0 <= 0x02000000) {
+				u32 r = (w0 - 0x00400000) / 0x400000;
+				u32 e1, va_hi, len;
+
+				if (r >= SVC_GR_CONTENT_RANKS || !gva[r])
+					continue;
+				if ((gpa + o) & (SEMA_PAGE_SIZE - 1))
+					continue;	/* entry0 at page start */
+				memcpy(&e1, buf + o + 4, 4);
+				va_hi = e1 & 0xff;
+				len = ((e1 >> 10) & 0x1fffff) << 2;
+				if (va_hi != 2 || (len & 3) || len < 4 ||
+				    len > SEMA_PAGE_SIZE)
+					continue;
+				if (!svc_gr_ring_tail_ok(buf + o + 8, 0x38))
+					continue;
+				svc_gr_content_record(svc_gr_content_ring[r],
+						      &svc_gr_content_nring[r],
+						      gpa + o, len);
+			}
+		}
+	}
+	kfree(buf);
+
+	for (i = 0; i < SVC_GR_CONTENT_RANKS; i++) {
+		u32 nr = svc_gr_content_nring[i];
+		u32 np = svc_gr_content_npush[i];
+		u32 a, b;
+
+		if (!gva[i] || !nr || !np)
+			continue;
+		for (a = 0; a < nr; a++) {
+			for (b = 0; b < np; b++) {
+				if (svc_gr_content_ring[i][a].len !=
+				    svc_gr_content_push[i][b].len)
+					continue;
+				if (!svc_gr_content_pin(
+					    vgpu, i, gva[i], ent[i],
+					    svc_gr_content_ring[i][a].pa_page,
+					    svc_gr_content_push[i][b].pa_page,
+					    svc_gr_content_ring[i][a].len))
+					continue;
+				break;
+			}
+			if (b < np)
+				break;
+		}
+	}
+}
+
+/*
  * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an
  * injected-pin miss, resolve a release's counter slot from the structural
  * translations (published page directories, self-discovered directory,
@@ -7322,39 +7827,58 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 	if (ch->userd_pa > SVC_GR_USERD_OFF) {
 		static const u64 deltas[] = { SVC_GR_USERD_OFF, 0x1000, 0x0,
 					      0x10000 };
-		int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS;
-		unsigned int di;
+		int caps[2];
+		unsigned int ci, di;
 
-		for (di = 0; di < ARRAY_SIZE(deltas); di++) {
-			u64 cand = ch->userd_pa - deltas[di];
+		/*
+		 * H2 (FORENSICS-v2): the registered aperture can be
+		 * mis-classified - v2 read the GR USERD through FB
+		 * (0xc0R000) and got 0/0, rc=-EAGAIN, while the pushbuffer
+		 * of the same boot is pinned in guest sysmem - so every
+		 * derived candidate is tried in the registered aperture
+		 * first and in the other one second. The entry0 + userd gate
+		 * below is unchanged; a wrong aperture only costs a failed
+		 * probe.
+		 */
+		caps[0] = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS;
+		caps[1] = caps[0] == SVC_APERTURE_FB ? SVC_APERTURE_SYS
+						     : SVC_APERTURE_FB;
+		for (ci = 0; ci < 2; ci++) {
+			int cap = caps[ci];
 
-			if (ch->userd_pa <= deltas[di])
-				continue;
-			if (svc_read_ap(vgpu, cand, cap, e, sizeof(e)))
-				continue;
-			memcpy(&e0, e, 4);
-			memcpy(&e1, e + 4, 4);
-			va = ((u64)(e1 & 0xff) << 32) |
-			     (u64)(e0 & 0xfffffffc);
-			len = ((e1 >> 10) & 0x1fffff) << 2;
-			if ((va & 0xfffff) || va < SEMA_VA_LO ||
-			    va >= SEMA_VA_HI || len < 4 || len > 0x10000)
-				continue;
-			if (svc_userd_probe(vgpu, ch->userd_pa, cap,
-					    ch->gpfifo_entries, &put, &get))
-				continue;
-			svc_pin_store(ch->gpfifo_va, cand, cap);
-			svc_lat_store(ch->gpfifo_va, cand, cap);
-			ch->userd_as = cap;
-			ch->userd_ok = true;
-			svc_gr_ring_base = cand;
-			set_bit(ch->gr_index, &svc_gr_ring_done);
-			svc_ring_n++;
-			pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n",
-				ch->h_object, ch->gr_index,
-				ch->gpfifo_va, cand, cap, deltas[di], va,
-				len, put, get);
-			return;
+			for (di = 0; di < ARRAY_SIZE(deltas); di++) {
+				u64 cand = ch->userd_pa - deltas[di];
+
+				if (ch->userd_pa <= deltas[di])
+					continue;
+				if (svc_read_ap(vgpu, cand, cap, e, sizeof(e)))
+					continue;
+				memcpy(&e0, e, 4);
+				memcpy(&e1, e + 4, 4);
+				va = ((u64)(e1 & 0xff) << 32) |
+				     (u64)(e0 & 0xfffffffc);
+				len = ((e1 >> 10) & 0x1fffff) << 2;
+				if ((va & 0xfffff) || va < SEMA_VA_LO ||
+				    va >= SEMA_VA_HI || len < 4 ||
+				    len > 0x10000)
+					continue;
+				if (svc_userd_probe(vgpu, ch->userd_pa, cap,
+						    ch->gpfifo_entries, &put,
+						    &get))
+					continue;
+				svc_pin_store(ch->gpfifo_va, cand, cap);
+				svc_lat_store(ch->gpfifo_va, cand, cap);
+				ch->userd_as = cap;
+				ch->userd_ok = true;
+				svc_gr_ring_base = cand;
+				set_bit(ch->gr_index, &svc_gr_ring_done);
+				svc_ring_n++;
+				pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n",
+					ch->h_object, ch->gr_index,
+					ch->gpfifo_va, cand, cap, deltas[di],
+					va, len, put, get);
+				return;
+			}
 		}
 	}
 
@@ -7445,6 +7969,39 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 		}
 	}
 
+	/*
+	 * H1: kernel-native trusted translation for the ring VA. The
+	 * driver-published VAS walk (with the self-discovered directory and
+	 * the targeted rm_mem fallback behind it) is the deterministic
+	 * guest-truth source that resolved the CE pool page in v2
+	 * (`pool-pa ... src=trusted`, 11-hostdmesg-v2-r1.log:2313), and it
+	 * is tried here for the ring: v2 has no pin for 0x200200000+R*0x3000
+	 * and the FB convention candidates read zeros. The same entry0 +
+	 * readable-USERD gate as everywhere else applies, so a wrong walk
+	 * result costs a failed probe, not a pin.
+	 */
+	{
+		u64 wpa = 0;
+		int wap = SVC_APERTURE_SYS;
+
+		if (!svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap) &&
+		    svc_gr_ring_candidate(vgpu, ch, wpa, wap, &va, &len, &put,
+					  &get)) {
+			svc_pin_store(ch->gpfifo_va, wpa, wap);
+			svc_lat_store(ch->gpfifo_va, wpa, wap);
+			ch->userd_pa = wpa + SVC_GR_USERD_OFF;
+			ch->userd_as = wap;
+			ch->userd_ok = true;
+			svc_gr_ring_base = wpa;
+			set_bit(ch->gr_index, &svc_gr_ring_done);
+			svc_ring_n++;
+			pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n",
+				ch->h_object, ch->gr_index, ch->gpfifo_va,
+				wpa, wap, va, len, put, get);
+			return;
+		}
+	}
+
 	if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) {
 		u8 d1[8] = { 0 }, d2[8] = { 0 };
 		int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS;
@@ -7547,8 +8104,6 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
  * carries the push-buffer VA, the base page was found by content and its
  * release names the channel's slot (4 bytes per rank).
  */
-static unsigned long svc_pb_done;	/* CE ranks whose push buffer is pinned */
-
 static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
 {
 	u64 hit_pa = 0, va = 0;
@@ -7594,11 +8149,12 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
 
 	/*
 	 * The pinned pairs are verified samples of UVM's own page directory;
-	 * let the directory discovery run again with them, so the walk can
-	 * translate the parts of the pool no base page describes.
+	 * the next discovery call sees the grown sample set as fresh evidence
+	 * and re-runs the scan (svc_pd_discover's attempt/evidence gates keep
+	 * that bounded), so the walk can translate the parts of the pool no
+	 * base page describes. On the old one-shot latch this pairing was the
+	 * only re-arm; the retryable discovery no longer needs it.
 	 */
-	if (!svc_pd_found)
-		svc_pd_tried = 0;
 
 	pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n",
 		ch->h_object, ch->ce_index, want, va, hit_pa, len);
@@ -7834,6 +8390,114 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 }
 
 /*
+ * Multi-page content rescue run (ISSUE-0001 durable direction).
+ *
+ * svc_rescue_try() substitutes exactly one candidate page and requires every
+ * other page of the entry to resolve through svc_lookup_trusted(). For a
+ * multi-page push whose pages have no translations yet that precondition can
+ * never hold: the stalled-boot class reads the CE entry at va=0x120021db8
+ * (len=12816, four pages, release #39 at the last page) through a window
+ * guess that is wrong for the entry page while the three continuation pages
+ * are missing from every trusted source; the content scan finds the
+ * release-group page (the matcher's candidate) but every single-substitution
+ * candidate is rejected because the untranslated pages are exactly the ones
+ * that need a candidate. The userspace pin stream was the only supplier of
+ * those translations (SA-STALL 3.2/3.4; 17-host-m84-r1.txt:841-842).
+ *
+ * A pushbuffer allocation is physically contiguous, so once content has
+ * anchored ONE page of the entry, the remaining pages sit at the same signed
+ * page delta from the anchor in one of the two directions. Observed both
+ * directions in the evidence: rank-2's consecutive release pages descend by
+ * 0x1000 per VA page ascending (17-host-m84-r1.txt:3-13: 0x1232b8000 ->
+ * 0x1232b7000 -> ... -> 0x1232b0000), i.e. dir=-1; other boots pair the
+ * window base forward (svc_pb_pair's convention), i.e. dir=+1. Try both:
+ *   dir=+1: VA page anchor_va + k*PAGE  -> PA anchor_pa + k*PAGE
+ *   dir=-1: VA page anchor_va + k*PAGE  -> PA anchor_pa - k*PAGE
+ * Read every page of the entry directly from the candidate PAs, validate the
+ * assembled buffer exactly as svc_rescue_try() does (full method-stream parse
+ * + this channel's exact next release), and only then pin every page of the
+ * run. A failed direction writes nothing, so a wrong anchor or direction
+ * cannot poison the pin table; a passed validation is content proof for
+ * every page, strictly stronger than the watcher pin it replaces.
+ *
+ * Bounded: entry_len <= SVC_PUSH_MAX, at most SVC_RESCUE_RUN_MAX pages
+ * (128 KB pushes fit), one direct read per page. Service-thread only.
+ */
+#define SVC_RESCUE_RUN_MAX	32	/* pages one rescue run may pin */
+
+static int svc_rescue_run_try(struct nv_vgpu_device *vgpu,
+			      struct svc_channel *ch, u64 entry_va,
+			      u32 entry_len, u64 anchor_pa,
+			      u64 anchor_va_page, int dir)
+{
+	u64 pin_va[SVC_RESCUE_RUN_MAX];
+	u64 pin_pa[SVC_RESCUE_RUN_MAX];
+	u8 *full;
+	u32 npages = 0, off = 0;
+	int rc = 0;
+
+	if (!entry_len || (dir != 1 && dir != -1))
+		return -EINVAL;
+
+	full = kmalloc(entry_len, GFP_KERNEL);
+	if (!full)
+		return -ENOMEM;
+
+	while (off < entry_len) {
+		u64 va = entry_va + off;
+		u64 va_page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+		size_t chunk = min_t(size_t, entry_len - off,
+				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
+		s64 delta = (s64)(va_page - anchor_va_page) * dir;
+		u64 pa;
+
+		if (npages >= SVC_RESCUE_RUN_MAX) {
+			rc = -E2BIG;
+			break;
+		}
+		if (delta < 0 && (u64)(-delta) > anchor_pa) {
+			rc = -EINVAL;
+			break;
+		}
+		pa = (u64)((s64)anchor_pa + delta);
+		if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) {
+			rc = -EINVAL;
+			break;
+		}
+		if (svc_read(vgpu, pa | (va & (SEMA_PAGE_SIZE - 1)),
+			     full + off, chunk)) {
+			rc = -EIO;
+			break;
+		}
+		pin_va[npages] = va_page;
+		pin_pa[npages] = pa;
+		npages++;
+		off += chunk;
+	}
+
+	if (!rc && svc_stream_full_ok(full, entry_len) &&
+	    svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload,
+				  full, entry_len)) {
+		u32 i;
+
+		for (i = 0; i < npages; i++) {
+			svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS);
+			svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS);
+			svc_mark_hot(pin_pa[i]);
+		}
+		svc.resc_run++;
+		pr_info("sema-svc: content rescue run ch=0x%x rank=%u entry_va=0x%llx len=%u want=%u dir=%d anchor=0x%llx pages=%u (pinned from content)\n",
+			ch->h_object, ch->ce_index, entry_va, entry_len,
+			ch->expected_sem_payload, dir, anchor_pa, npages);
+		rc = 0;
+	} else if (!rc) {
+		rc = -ENOENT;
+	}
+	kfree(full);
+	return rc;
+}
+
+/*
  * Scan a guest-RAM range descending for a page that validates as this entry's
  * push (method stream + expected release payload), pinning it and returning 0
  * on the first hit. `hi` and `lo` are inclusive chunk start addresses.
@@ -7862,6 +8526,21 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu,
 						     &ch->content_max);
 			total += ncand;
 			for (n = 0; n < ncand; n++) {
+				/*
+				 * Run first: when the entry spans pages with
+				 * no translations yet, only the multi-page
+				 * assembly can validate the candidate (see
+				 * svc_rescue_run_try); the single-page try
+				 * below stays as the fallback for entries
+				 * whose other pages are already trusted.
+				 */
+				if (!svc_rescue_run_try(vgpu, ch, entry_va,
+							entry_len, cand[n],
+							cand_va[n], 1) ||
+				    !svc_rescue_run_try(vgpu, ch, entry_va,
+							entry_len, cand[n],
+							cand_va[n], -1))
+					return 0;
 				if (!svc_rescue_try(vgpu, ch, entry_va,
 						    entry_len, cand[n],
 						    cand_va[n], n, ncand))
@@ -7927,6 +8606,22 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 		    entry_va < svc_pb[i].va)
 			continue;
 		cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va);
+		/*
+		 * The window base is guest-truth only for its first page; a
+		 * multi-page entry may continue in either physical direction
+		 * (observed both). Try the whole run first: it pins every page
+		 * from content and needs no pre-existing translations, which
+		 * the single-page try below does.
+		 */
+		if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len,
+					cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+					entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+					1) ||
+		    !svc_rescue_run_try(vgpu, ch, entry_va, entry_len,
+					cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+					entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+					-1))
+			return 0;
 		if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand_pa,
 				    entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
 				    i, svc_pb_n))
@@ -7962,7 +8657,13 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 						     cand_va, 0,
 						     &ch->content_max);
 			for (n = 0; n < ncand; n++) {
-				if (!svc_rescue_try(vgpu, ch, entry_va,
+				if (!svc_rescue_run_try(vgpu, ch, entry_va,
+							entry_len, cand[n],
+							cand_va[n], 1) ||
+				    !svc_rescue_run_try(vgpu, ch, entry_va,
+							entry_len, cand[n],
+							cand_va[n], -1) ||
+				    !svc_rescue_try(vgpu, ch, entry_va,
 						    entry_len, cand[n],
 						    cand_va[n], n, ncand)) {
 					kfree(buf);
@@ -10484,6 +11185,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 	n_gr = SVC_GR_COUNTER_TOTAL;
 
 	/*
+	 * H1 last resort: one self-gated content-discovery pass for the GR
+	 * ctx-init rings before this pass's channel loop. The pins it
+	 * installs (ring VA + push VA pages, both validated from content and
+	 * the USERD) are picked up by svc_gr_ring_autopin's existing pin
+	 * branch and then by the pump, exactly like a watcher pin - the
+	 * difference is that the content is guest truth and needs no
+	 * userspace pipeline.
+	 */
+	svc_gr_content_discover(vgpu);
+
+	/*
 	 * P6b wave 1.5: a hold whose channel is absent from this pass's
 	 * snapshot - or belongs to another device than the one being pumped -
 	 * cannot be resolved through the entry loop. Once the work item has an
@@ -10518,8 +11230,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		 * remains.
 		 */
 		if (!ch->engine_type) {
+			u32 pput = 0;
+
 			svc_counter_autoreg(vgpu, ch, n_gr);
 			svc_counter_flush_pending(vgpu, ch);
+			/*
+			 * H3 (FORENSICS-v2): the end-of-pass publish below is
+			 * unreachable on the passes that early-`continue`
+			 * before it (USERD not resolvable, GP_PUT unreadable)
+			 * even though the counter slot is registered and the
+			 * client's tracker waits on it - v2 had pub_calls=0
+			 * with 20/20 slots registered and the GR ctx-init
+			 * releases unread. Publish here, next to the safety
+			 * net, from the registered USERD when it is readable.
+			 * The value is a real GPFIFO put read from the
+			 * channel's own control block, never a guess, and the
+			 * target is the proof-gated slot registration
+			 * (injected pin, or the read-back-validated derive of
+			 * svc_counter_autoreg). Skip while a relay hold owns
+			 * the channel, exactly like the end-of-pass call.
+			 */
+			if (ch->counter_pa && ch->userd_pa &&
+			    !(svc_fwd_hold.active &&
+			      svc_fwd_hold.vgpu == vgpu &&
+			      svc_fwd_hold.ch_handle == ch->h_object) &&
+			    !svc_read_ap(vgpu,
+					 ch->userd_pa + SVC_USERD_GP_PUT,
+					 ch->userd_as, &pput, sizeof(pput)))
+				svc_counter_publish(vgpu, ch, pput);
 		}
 
 		/*
@@ -11710,8 +12448,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		 * safety net at the top of the pass, so this must not
 		 * re-register: once per pass is enough and the derivation is
 		 * not free.
+		 *
+		 * H3 (FORENSICS-v2): GR channels publish in the safety-net
+		 * block at the top of the pass - the early `continue`s above
+		 * make this call unreachable exactly when the ring or USERD
+		 * cannot be read, which is the case the publish must still
+		 * cover. Keep this site for the CE channels only, so a GR
+		 * pass does not publish twice.
 		 */
-		if (!held)
+		if (!held && ch->engine_type)
 			svc_counter_publish(vgpu, ch, put);
 
 		spin_lock_irqsave(&svc.chan_lock, flags);