vgsp-sema: multi-page content rescue pins a whole CE push from RAM content (ISSUE-0001)

svc_rescue_try() substitutes exactly one candidate page and requires every
other page of the entry to resolve through svc_lookup_trusted(). For a
multi-page push whose pages have no translations yet that precondition can
never hold: the content scan finds the release-group page by content, but
the pages that need a candidate are exactly the ones missing from every
trusted source. That is the stalled-boot class named by SA-STALL (entry
va=0x120021db8, len=12816, release #39; 17-host-m84-r1.txt:841-842), where
the userspace pin stream was the only supplier of those translations.

Add svc_rescue_run_try(): once content has anchored one page of the entry
(release-group candidate from the matcher, or the verified window base),
read every page of the entry directly from the same signed page delta in
both physical directions, validate the assembled buffer exactly like the
single-page rescue (full method-stream parse + the channel's exact next
release), and only then pin every page of the run. A failed direction
pins nothing; a passed validation is content proof for every page. Wire it
into the svc_pb fast path, the hot-page scan and the descending RAM scan,
ahead of the unchanged single-substitution try.

This makes the pump able to consume a starved CE push with no watcher pins
and no pre-existing per-page translations; the watchers become pure
accelerators. New counter resc_run is printed when non-zero.

Evidence: ISSUE-0001, SA-STALL 3.2/3.4; 17-host-m104-r1.txt entry[38] and
17-host-m84-r1.txt rescue rejects.

1 files changed, 151 insertions, 1 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 8d223d8..e69c46c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -529,6 +529,8 @@ struct sema_svc {
 	u64			gpfifo_skipped;		/* stale entries consumed */
 	u64			gpfifo_rescued;		/* unreadable push made readable
 						 * by the content rescue */
+	u64			resc_run;		/* pushes pinned by the multi-page
+						 * content rescue run */
 	u64			resc_resync;		/* expectation re-synced down to
 						 * the content watermark */
 	u64			releases_seen;
@@ -2916,6 +2918,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n",
 		svc.counter_pub_calls, svc.gr_rel_applied,
 		svc.sem_write_reresolve, svc.pushpg_fallback);
+	if (svc.resc_run)
+		pr_info("sema-svc:   rescue_run=%llu (multi-page content pins)\n",
+			svc.resc_run);
 	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n",
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
 		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
@@ -7927,6 +7932,114 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 }
 
 /*
+ * Multi-page content rescue run (ISSUE-0001 durable direction).
+ *
+ * svc_rescue_try() substitutes exactly one candidate page and requires every
+ * other page of the entry to resolve through svc_lookup_trusted(). For a
+ * multi-page push whose pages have no translations yet that precondition can
+ * never hold: the stalled-boot class reads the CE entry at va=0x120021db8
+ * (len=12816, four pages, release #39 at the last page) through a window
+ * guess that is wrong for the entry page while the three continuation pages
+ * are missing from every trusted source; the content scan finds the
+ * release-group page (the matcher's candidate) but every single-substitution
+ * candidate is rejected because the untranslated pages are exactly the ones
+ * that need a candidate. The userspace pin stream was the only supplier of
+ * those translations (SA-STALL 3.2/3.4; 17-host-m84-r1.txt:841-842).
+ *
+ * A pushbuffer allocation is physically contiguous, so once content has
+ * anchored ONE page of the entry, the remaining pages sit at the same signed
+ * page delta from the anchor in one of the two directions. Observed both
+ * directions in the evidence: rank-2's consecutive release pages descend by
+ * 0x1000 per VA page ascending (17-host-m84-r1.txt:3-13: 0x1232b8000 ->
+ * 0x1232b7000 -> ... -> 0x1232b0000), i.e. dir=-1; other boots pair the
+ * window base forward (svc_pb_pair's convention), i.e. dir=+1. Try both:
+ *   dir=+1: VA page anchor_va + k*PAGE  -> PA anchor_pa + k*PAGE
+ *   dir=-1: VA page anchor_va + k*PAGE  -> PA anchor_pa - k*PAGE
+ * Read every page of the entry directly from the candidate PAs, validate the
+ * assembled buffer exactly as svc_rescue_try() does (full method-stream parse
+ * + this channel's exact next release), and only then pin every page of the
+ * run. A failed direction writes nothing, so a wrong anchor or direction
+ * cannot poison the pin table; a passed validation is content proof for
+ * every page, strictly stronger than the watcher pin it replaces.
+ *
+ * Bounded: entry_len <= SVC_PUSH_MAX, at most SVC_RESCUE_RUN_MAX pages
+ * (128 KB pushes fit), one direct read per page. Service-thread only.
+ */
+#define SVC_RESCUE_RUN_MAX	32	/* pages one rescue run may pin */
+
+static int svc_rescue_run_try(struct nv_vgpu_device *vgpu,
+			      struct svc_channel *ch, u64 entry_va,
+			      u32 entry_len, u64 anchor_pa,
+			      u64 anchor_va_page, int dir)
+{
+	u64 pin_va[SVC_RESCUE_RUN_MAX];
+	u64 pin_pa[SVC_RESCUE_RUN_MAX];
+	u8 *full;
+	u32 npages = 0, off = 0;
+	int rc = 0;
+
+	if (!entry_len || (dir != 1 && dir != -1))
+		return -EINVAL;
+
+	full = kmalloc(entry_len, GFP_KERNEL);
+	if (!full)
+		return -ENOMEM;
+
+	while (off < entry_len) {
+		u64 va = entry_va + off;
+		u64 va_page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+		size_t chunk = min_t(size_t, entry_len - off,
+				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
+		s64 delta = (s64)(va_page - anchor_va_page) * dir;
+		u64 pa;
+
+		if (npages >= SVC_RESCUE_RUN_MAX) {
+			rc = -E2BIG;
+			break;
+		}
+		if (delta < 0 && (u64)(-delta) > anchor_pa) {
+			rc = -EINVAL;
+			break;
+		}
+		pa = (u64)((s64)anchor_pa + delta);
+		if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) {
+			rc = -EINVAL;
+			break;
+		}
+		if (svc_read(vgpu, pa | (va & (SEMA_PAGE_SIZE - 1)),
+			     full + off, chunk)) {
+			rc = -EIO;
+			break;
+		}
+		pin_va[npages] = va_page;
+		pin_pa[npages] = pa;
+		npages++;
+		off += chunk;
+	}
+
+	if (!rc && svc_stream_full_ok(full, entry_len) &&
+	    svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload,
+				  full, entry_len)) {
+		u32 i;
+
+		for (i = 0; i < npages; i++) {
+			svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS);
+			svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS);
+			svc_mark_hot(pin_pa[i]);
+		}
+		svc.resc_run++;
+		pr_info("sema-svc: content rescue run ch=0x%x rank=%u entry_va=0x%llx len=%u want=%u dir=%d anchor=0x%llx pages=%u (pinned from content)\n",
+			ch->h_object, ch->ce_index, entry_va, entry_len,
+			ch->expected_sem_payload, dir, anchor_pa, npages);
+		rc = 0;
+	} else if (!rc) {
+		rc = -ENOENT;
+	}
+	kfree(full);
+	return rc;
+}
+
+/*
  * Scan a guest-RAM range descending for a page that validates as this entry's
  * push (method stream + expected release payload), pinning it and returning 0
  * on the first hit. `hi` and `lo` are inclusive chunk start addresses.
@@ -7955,6 +8068,21 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu,
 						     &ch->content_max);
 			total += ncand;
 			for (n = 0; n < ncand; n++) {
+				/*
+				 * Run first: when the entry spans pages with
+				 * no translations yet, only the multi-page
+				 * assembly can validate the candidate (see
+				 * svc_rescue_run_try); the single-page try
+				 * below stays as the fallback for entries
+				 * whose other pages are already trusted.
+				 */
+				if (!svc_rescue_run_try(vgpu, ch, entry_va,
+							entry_len, cand[n],
+							cand_va[n], 1) ||
+				    !svc_rescue_run_try(vgpu, ch, entry_va,
+							entry_len, cand[n],
+							cand_va[n], -1))
+					return 0;
 				if (!svc_rescue_try(vgpu, ch, entry_va,
 						    entry_len, cand[n],
 						    cand_va[n], n, ncand))
@@ -8020,6 +8148,22 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 		    entry_va < svc_pb[i].va)
 			continue;
 		cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va);
+		/*
+		 * The window base is guest-truth only for its first page; a
+		 * multi-page entry may continue in either physical direction
+		 * (observed both). Try the whole run first: it pins every page
+		 * from content and needs no pre-existing translations, which
+		 * the single-page try below does.
+		 */
+		if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len,
+					cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+					entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+					1) ||
+		    !svc_rescue_run_try(vgpu, ch, entry_va, entry_len,
+					cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+					entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+					-1))
+			return 0;
 		if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand_pa,
 				    entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
 				    i, svc_pb_n))
@@ -8055,7 +8199,13 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 						     cand_va, 0,
 						     &ch->content_max);
 			for (n = 0; n < ncand; n++) {
-				if (!svc_rescue_try(vgpu, ch, entry_va,
+				if (!svc_rescue_run_try(vgpu, ch, entry_va,
+							entry_len, cand[n],
+							cand_va[n], 1) ||
+				    !svc_rescue_run_try(vgpu, ch, entry_va,
+							entry_len, cand[n],
+							cand_va[n], -1) ||
+				    !svc_rescue_try(vgpu, ch, entry_va,
 						    entry_len, cand[n],
 						    cand_va[n], n, ncand)) {
 					kfree(buf);