vgsp: verify the scrub ring ledger before authoring (P0 for the X/GL wedge, CONT.427)

- Resolve the channel GPFIFO ring page through the GPU page tables (the
  same resolver the stash geometry path trusts) instead of deriving it as
  "hsem page - entries*8"; that subtraction is only correct when the
  buffer's GPFIFO and sema pages are PA-contiguous, and it could land on a
  stale or dense page (r1775/r1777 boots).
- Validate the first read with the strict GPFIFO entry shape (flag bits,
  length field, push VA bounded around the channel's own gpfifo) plus a
  clean zero terminus (the next slot and the following 32 slots must be
  zero). An all-zero page, a dense data page or a dirty tail is not
  trusted as a submission ledger.
- Detect a guest scrubber reset by a regressed finish word (or fin
  returning to 0 while a release was authored) and reseed the ledger
  instead of re-applying the pre-reset absolute count - that
  re-application underflowed the guest's scrub-list size (r1775 c6:
  4076 into an empty list -> 0xFFFFFFFFFFFFF014 -> the
  scrubCheckAndWaitForSize spin, _nv052600rm+0x7b).
- Never author the finish word from an unverified or unresolved ring:
  fin stays a no-op. A blocked scrub wait is recoverable; an underflowed
  list is not. A/B: sema_scrub_ring=0 skips the ledger entirely.

Live validation of this build is pending (6-cycle wedge acceptance +
GZDoom run).

1 files changed, 198 insertions, 30 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 3225008..9d9f6c8 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1122,8 +1122,15 @@ struct svc_scrub {
 	 */
 	u32	ring_cum;	/* cumulative pushes counted from the ring */
 	unsigned long ring_at;	/* last ring read (rate limit) */
+	unsigned long ring_valid_at;	/* last first-read validation attempt */
+	u64	ring_pa;	/* ring page resolved in the latest read */
+	bool	ring_page_ok;	/* GPFIFO VA resolved this pass */
+	bool	ring_read_ok;	/* the last ring read completed */
 	bool	ring_ready;	/* snapshot initialized for this geometry */
-	bool	ring_logged;	/* one-shot ring log */
+	bool	ring_valid;	/* first-read prefix+terminus accepted */
+	bool	ring_logged;	/* one-shot ring ledger log */
+	bool	ring_release_logged;	/* one-shot ring release log */
+	u32	ring_resets;	/* bounded reset-log budget */
 };
 /*
  * One scrubber record per guest-channel buffer: every guest's PMA has its
@@ -1177,6 +1184,17 @@ module_param_named(sema_scrub_ramp, gsp_sema_scrub_ramp, bool, 0644);
 MODULE_PARM_DESC(sema_scrub_ramp,
 	"scrub finish release: 1 = real submitted counts only (default), 0 = legacy all-ones");
 
+/*
+ * r1779 (V13): P0 for the per-boot X/GL wedge. The ring ledger must never
+ * author a fin value for a ring it has not verified. sema_scrub_ring=0 is the
+ * A/B control: the ledger is skipped entirely and fin stays a no-op (a
+ * blocked scrub wait is recoverable; an underflowed scrubListSize is not).
+ */
+static bool gsp_sema_scrub_ring = true;
+module_param_named(sema_scrub_ring, gsp_sema_scrub_ring, bool, 0644);
+MODULE_PARM_DESC(sema_scrub_ring,
+	"scrub fin ledger from the channel GPFIFO ring: 1 = enabled (default), 0 = A/B off");
+
 static u32 svc_scrub_fin_next(struct svc_scrub *s, u32 cur)
 {
 	u32 base;
@@ -2178,9 +2196,21 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
 #define SVC_SCRUB_RING_ENTRIES	4096u
 #define SVC_SCRUB_RING_BYTES	(SVC_SCRUB_RING_ENTRIES * 8u)
 #define SVC_SCRUB_RING_RATE_MS	250u
+/* r1779 (V13): re-run the prefix/terminus validation at most this often. */
+#define SVC_SCRUB_RING_VALID_MS	500u
+/* r1779 (V13): a scrub push must lie within this window of the GPFIFO VA. */
+#define SVC_SCRUB_RING_PUSH_WIN	0x1000000u
+
+/*
+ * Per-record entry snapshot (lo | hi<<32), lazily allocated, freed at
+ * service stop. The hi dword carries the GPFIFO flags/length, so the whole
+ * 8-byte entry is snapshotted and validated (r1779/V13).
+ */
+static u64 *svc_scrub_ring_snap[SVC_MAX_GUESTS * SVC_SCRUB_RECS];
 
-/* Per-record push_lo snapshot (lazily allocated, freed at service stop). */
-static u32 *svc_scrub_ring_snap[SVC_MAX_GUESTS * SVC_SCRUB_RECS];
+/* Defined with the stash geometry resolver below; reused for the GPFIFO VA. */
+static bool svc_scrub_resolve_va(struct nv_vgpu_device *vgpu, u64 va,
+				 u64 *pa, int *ap, u32 *hvas);
 
 static inline bool svc_scrub_push_lo(u32 lo)
 {
@@ -2188,6 +2218,84 @@ static inline bool svc_scrub_push_lo(u32 lo)
 }
 
 /*
+ * r1779 (V13): strict GPFIFO entry shape for the CeUtils scrubber ledger.
+ * The permissive svc_scrub_push_lo() above only recognises the low dword's
+ * 2 MiB push window, which a dense data page can satisfy. Real scrubber
+ * entries are {push_va, 0x00006001-class flags} (r1768b dump): require the
+ * hardware flag bits, a plausible length field, and a push VA in a bounded
+ * window around this channel's own GPFIFO VA.
+ */
+static bool svc_scrub_entry_strict(struct svc_scrub *s, u32 lo, u32 hi)
+{
+	s32 off;
+	u32 len;
+
+	if ((hi & 3u) != 1u)
+		return false;
+	len = ((hi >> 10) & 0x1fffffu) << 2;
+	if (len < 4u || len > 0x10000u)
+		return false;
+	if (lo & 3u)
+		return false;
+	off = (s32)(lo - (u32)s->gpfifo_va);
+	if (off > (s32)SVC_SCRUB_RING_PUSH_WIN ||
+	    off < -(s32)SVC_SCRUB_RING_PUSH_WIN)
+		return false;
+	return true;
+}
+
+/*
+ * r1779 (V13): resolve the channel's GPFIFO ring page through the GPU page
+ * tables instead of deriving it as "hsem page - entries*8". The old
+ * arithmetic is only correct when the channel buffer's GPFIFO and sema
+ * pages are PA-contiguous; the GPFIFO VA is a different PTE, and in the
+ * r1775/r1777 boots the subtraction could land on a stale/dense page. The
+ * walk here is the same one the stash geometry path already trusts.
+ */
+static bool svc_scrub_ring_page(struct nv_vgpu_device *vgpu,
+				struct svc_scrub *s, u64 *ring_pa)
+{
+	u64 wpa = 0;
+	int wap = SVC_APERTURE_SYS;
+	u32 hv = 0;
+
+	if (!s->gpfifo_va)
+		return false;
+	if (!svc_scrub_resolve_va(vgpu, s->gpfifo_va, &wpa, &wap, &hv) || !wpa)
+		return false;
+	if (svc_ap_of(wap) != SVC_APERTURE_SYS)
+		return false;
+	*ring_pa = wpa & ~0xfffull;
+	return true;
+}
+
+/*
+ * r1779 (V13): first-read validation. The guest pushes sequentially, so a
+ * genuine ring is a contiguous strict-valid prefix followed by a fully-zero
+ * slot and at least the next 32 zero slots (the tail is only written after
+ * a wrap). Anything else - an all-zero page, a dense data page, a stale
+ * ring with a dirty tail - is not trusted as a submission ledger.
+ */
+static bool svc_scrub_ring_validate(struct svc_scrub *s, const u64 *snap,
+				    u32 *count)
+{
+	u32 n = 0, z;
+	bool tail = true;
+
+	while (n < SVC_SCRUB_RING_ENTRIES &&
+	       svc_scrub_entry_strict(s, (u32)snap[n], (u32)(snap[n] >> 32)))
+		n++;
+	for (z = n; z <= n + 32 && z < SVC_SCRUB_RING_ENTRIES; z++) {
+		if (snap[z]) {
+			tail = false;
+			break;
+		}
+	}
+	*count = n;
+	return n > 0 && n < SVC_SCRUB_RING_ENTRIES && tail;
+}
+
+/*
  * A scanner vmap candidate can resolve the channel GPFIFO ring page
  * (channel base + 0x64000) and register it as if it were the sema/fin
  * page. Authoring the all-ones release pair there overwrites GPFIFO entry
@@ -2220,7 +2328,12 @@ static bool svc_scrub_is_ring_page(struct svc_scrub *s)
 static void svc_scrub_ring_reset(struct svc_scrub *s)
 {
 	s->ring_cum = 0;
+	s->ring_at = 0;
+	s->ring_valid_at = 0;
 	s->ring_ready = false;
+	s->ring_valid = false;
+	s->ring_page_ok = false;
+	s->ring_read_ok = false;
 }
 
 /*
@@ -2232,12 +2345,13 @@ static void svc_scrub_ring_reset(struct svc_scrub *s)
  */
 static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s)
 {
-	u32 *snap = svc_scrub_ring_snap[s - svc_scrubs];
+	u64 *snap = svc_scrub_ring_snap[s - svc_scrubs];
 	u64 ring_pa;
 	u8 *chunk;
 	u32 i;
 	bool ok = true;
 
+	s->ring_read_ok = false;
 	if (!s->pa_ok || !s->entries || s->entries > SVC_SCRUB_RING_ENTRIES)
 		return s->ring_cum;
 	if (time_before(jiffies,
@@ -2245,7 +2359,7 @@ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s)
 		return s->ring_cum;
 	s->ring_at = jiffies;
 	if (!snap) {
-		snap = kzalloc(SVC_SCRUB_RING_ENTRIES * sizeof(u32), GFP_KERNEL);
+		snap = kzalloc(SVC_SCRUB_RING_ENTRIES * sizeof(u64), GFP_KERNEL);
 		if (!snap)
 			return s->ring_cum;
 		svc_scrub_ring_snap[s - svc_scrubs] = snap;
@@ -2253,7 +2367,7 @@ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s)
 	chunk = kmalloc(4096, GFP_KERNEL);
 	if (!chunk)
 		return s->ring_cum;
-	ring_pa = (s->hsem_pa & ~0xfffull) - (u64)s->entries * 8u;
+	ring_pa = s->ring_pa;
 	for (i = 0; i < SVC_SCRUB_RING_ENTRIES; i += 512) {
 		u32 j;
 
@@ -2263,37 +2377,54 @@ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s)
 			break;
 		}
 		for (j = 0; j < 512; j++) {
-			u32 lo = *(u32 *)(chunk + j * 8);
+			u64 e = *(u64 *)(chunk + j * 8);
+			u32 k = i + j;
 
-			if (!s->ring_ready) {
-				/*
-				 * First read for this geometry: the guest
-				 * pushes sequentially, so the highest valid
-				 * entry index + 1 is the push count. (Robust
-				 * to a clobbered entry 0 - only the maximum
-				 * matters.)
-				 */
-				if (svc_scrub_push_lo(lo) &&
-				    i + j + 1 > s->ring_cum)
-					s->ring_cum = i + j + 1;
-			} else if (lo != snap[i + j]) {
-				/* New or overwritten (wrap) push. */
-				if (svc_scrub_push_lo(lo))
-					s->ring_cum++;
-			}
-			snap[i + j] = lo;
+			/*
+			 * Incremental counting only after the first read was
+			 * validated: a new or overwritten (wrap) strict entry
+			 * is one push. Before validation the snapshot only
+			 * records content for the prefix check below.
+			 */
+			if (s->ring_valid && e != snap[k] &&
+			    svc_scrub_entry_strict(s, (u32)e,
+						   (u32)(e >> 32)))
+				s->ring_cum++;
+			snap[k] = e;
 		}
 	}
 	kfree(chunk);
 	if (!ok)
 		return s->ring_cum;
-	if (!s->ring_ready) {
+	s->ring_read_ok = true;
+	if (!s->ring_valid) {
+		u32 n = 0;
+
+		/*
+		 * Bound the validation cost: it reads only the in-memory
+		 * snapshot, but it runs from the per-pass authoring path, so
+		 * re-attempt at most every SVC_SCRUB_RING_VALID_MS. A fresh
+		 * geometry (ring_ready=false) validates immediately.
+		 */
+		if (s->ring_ready &&
+		    time_before(jiffies, s->ring_valid_at +
+				msecs_to_jiffies(SVC_SCRUB_RING_VALID_MS)))
+			return s->ring_cum;
+		s->ring_valid_at = jiffies;
 		s->ring_ready = true;
-		if (s->ring_cum && !s->ring_logged) {
+		if (!svc_scrub_ring_validate(s, snap, &n)) {
+			pr_info_ratelimited("sema-svc: scrub ring ledger disabled fin_pa=0x%llx ring_pa=0x%llx (prefix/terminus rejected n=%u)\n",
+					    s->fin_pa, ring_pa, n);
+			s->ring_cum = 0;
+			return 0;
+		}
+		s->ring_cum = n;
+		if (!s->ring_logged) {
 			s->ring_logged = true;
 			pr_info("sema-svc: scrub ring ledger fin_pa=0x%llx ring_pa=0x%llx pushes=%u\n",
 				s->fin_pa, ring_pa, s->ring_cum);
 		}
+		s->ring_valid = true;
 	}
 	return s->ring_cum;
 }
@@ -2303,16 +2434,53 @@ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s)
  * fabricates progress - the count is the guest's own submissions, so the
  * reported delta can never exceed the pending list (see the invariant
  * above). With no observed pushes the word is left untouched.
+ *
+ * r1779 (V13) P0 guards:
+ *  - a regressed fin word means the guest reset its scrubber object while
+ *    the resolved DA stayed the same; re-applying the pre-reset absolute
+ *    count underflows scrubListSize (r1775 c6: 4076 into an empty list ->
+ *    0xFFFFFFFFFFFFF014 -> scrubCheckAndWaitForSize spin, _nv052600rm+0x7b);
+ *  - the ring value is used only when the GPFIFO page resolved this pass
+ *    AND its first read passed the prefix/terminus validation this session.
  */
 static u32 svc_scrub_fin_next_ring(struct nv_vgpu_device *vgpu,
 				   struct svc_scrub *s, u32 cur)
 {
-	u32 ring = svc_scrub_ring_cum(vgpu, s);
-
+	u64 ring_pa = 0;
+	u32 ring;
+
+	if ((s32)(cur - s->fin_release) < 0 ||
+	    (cur == 0 && s->fin_release != 0)) {
+		if (s->ring_resets < 16) {
+			s->ring_resets++;
+			pr_info("sema-svc: scrub ring reset (fin regressed old=0x%x new=0x%x)\n",
+				s->fin_release, cur);
+		}
+		svc_scrub_ring_reset(s);
+		s->fin_release = cur;
+		return cur;		/* no write this pass */
+	}
+	if (!gsp_sema_scrub_ring)
+		return cur;
+	/* Match the ring read rate limit: no walk on a rate-limited pass. */
+	if (time_before(jiffies,
+			s->ring_at + msecs_to_jiffies(SVC_SCRUB_RING_RATE_MS)))
+		return cur;
+	if (!svc_scrub_ring_page(vgpu, s, &ring_pa)) {
+		s->ring_page_ok = false;
+		pr_info_ratelimited("sema-svc: scrub ring ledger disabled (ring walk failed) h=0x%x va=0x%llx\n",
+				    s->h_object, s->gpfifo_va);
+		return cur;
+	}
+	s->ring_page_ok = true;
+	s->ring_pa = ring_pa;
+	ring = svc_scrub_ring_cum(vgpu, s);
+	if (!s->ring_read_ok || !s->ring_valid)
+		return cur;		/* unverified ring: no ledger */
 	if (!ring || (s32)(ring - cur) <= 0)
 		return cur;		/* no pushes, or forward-only */
-	if (!s->ring_logged) {
-		s->ring_logged = true;
+	if (!s->ring_release_logged) {
+		s->ring_release_logged = true;
 		pr_info("sema-svc: scrub ring release fin_pa=0x%llx cum=%u (was 0x%x)\n",
 			s->fin_pa, ring, cur);
 	}