vgsp: sysmem GR USERD RAM scan (sliced, ring-entry validated) + unreadable-push demand regex (SESSION 153 WIP: liveness latch pending)

2 files changed, 277 insertions, 2 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 812cfea..8711517 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -554,6 +554,18 @@ struct svc_channel {
 	u8	gr_ram_refreshes;	/* budget refreshes granted so far */
 	unsigned long gr_ram_refresh_at;	/* next eligible refresh */
 	/*
+	 * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose
+	 * ring was content-validated in RAM (gr_ram_ring). Such a client's
+	 * USERD is a system-memory object like its ring (the kernel-RM/X GL
+	 * path), so the FB-shadow shape scan can never reach it and the pump
+	 * never iterates its entries. Same sliced/cursor discipline as the
+	 * ring scan; the cursor is persisted immediately because the
+	 * stalled-userd pump path skips the end-of-pass write-back.
+	 */
+	u64	gr_ram_userd_cursor;	/* resume point of the USERD scan */
+	u8	gr_ram_userd_passes;	/* slices walked so far */
+	bool	gr_ram_userd_logged;	/* one discovery/give-up line */
+	/*
 	 * r496e: true when the ring was found by the RAM content scan (a
 	 * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's
 	 * pushes are 2D/3D method streams the emulation does not execute;
@@ -1509,6 +1521,17 @@ static void svc_chan_rearm_rank(struct svc_channel *ch)
 		clear_bit(ch->ce_index, &svc_pb_done);
 	} else {
 		clear_bit(ch->gr_index, &svc_gr_ring_done);
+		/*
+		 * r914: re-arm the ring-anchored window pairing too. The GL
+		 * client re-registers with a NEW ring PA on every attempt,
+		 * and with only the ring bit cleared the stale first pair
+		 * (svc_pb_win_lookup returns the first slot match) kept
+		 * answering the push reads with the previous attempt's
+		 * pages - which the guest had since reused for kernel
+		 * structures (measured r913: the push read resolved to a
+		 * page of kernel pointers and every entry went unreadable).
+		 */
+		clear_bit(ch->gr_index, &svc_pb_gr_done);
 	}
 }
 
@@ -15682,6 +15705,11 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as,
 	return 0;
 }
 
+/* r912: guest-RAM USERD shape scan (defined next to the ring RAM scan). */
+static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
+				  struct svc_channel *ch, u64 *pa_out,
+				  u32 *put_out, u32 *get_out);
+
 /*
  * Locate a channel's USERD. The allocation parameters carry an offset that is
  * relative to a client pool the emulation cannot see, so it is only one
@@ -15907,6 +15935,33 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	}
 
 	/*
+	 * r912 (M-GL1.2): a GR channel whose ring was content-validated in
+	 * guest RAM (gr_ram_ring) has its USERD in guest RAM too - the
+	 * FB-shadow sweep below can never reach it. The guest-RAM shape
+	 * scan is sliced and budgeted like the ring scan; it validates
+	 * every entry the candidate's put claims against the ring window
+	 * before latching (see svc_gr_ram_userd_scan).
+	 */
+	if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok &&
+	    ch->gr_ram_userd_passes < svc_gr_ram_pass_budget()) {
+		u64 upa = 0;
+		u32 uput = 0, uget = 0;
+
+		if (svc_gr_ram_userd_scan(vgpu, ch, &upa, &uput, &uget)) {
+			ch->userd_pa = upa;
+			ch->userd_as = SVC_APERTURE_SYS;
+			ch->userd_ok = true;
+			ch->last_get = 0;
+			*put = uput;
+			*get = uget;
+			pr_info("sema-chan: GR USERD h=0x%x rank=%u found by RAM scan -> pa=0x%llx (put=%u get=%u%s; entries ring-validated)\n",
+				ch->h_object, ch->gr_index, upa, uput, uget,
+				uget < uput ? "" : ", idle");
+			return 0;
+		}
+	}
+
+	/*
 	 * r496b: no candidate matched. For a GR channel whose ring is known
 	 * (translated and pinned - the sysmem content scan or the VRAM
 	 * conventions), the USERD may still be elsewhere: the X/GL driver
@@ -17728,6 +17783,190 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 }
 
 /*
+ * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring
+ * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan).
+ * Such a client's USERD is a system-memory object like its ring, so the
+ * FB-shadow shape scan inside svc_chan_userd can never reach it: the
+ * registered userd value is pool-relative, every FB candidate reads
+ * garbage, and the pump never iterates the GPFIFO entries ("unregistered
+ * (submissions=0)"). Scan guest RAM for the USERD control block shape
+ * (GET at +0x88, PUT at +0x8c - SVC_USERD_GP_*), with the same sliced,
+ * cursor-resumed budget as the ring scan (one SVC_GR_RAM_SLICE slice per
+ * pass, svc_gr_ram_pass_budget() slices before giving up).
+ *
+ * A candidate is latched ONLY when every entry its PUT claims decodes to
+ * a pushbuffer VA inside the ring's allocation window: the entries are
+ * read at the ring VA's own offset (entry k at ring VA + k*8) and must
+ * land within +/-SVC_GR_RAM_WIN/2 of the ring VA with a sane length -
+ * the same gate the FB shape scan applies, so a coincidental put/get
+ * pair costs a rejected candidate, never a wrong latch. Among valid
+ * candidates a block with work PENDING (get < put) wins: the live block
+ * of a stuck client always shows its unconsumed submission.
+ */
+static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
+				  struct svc_channel *ch, u64 *pa_out,
+				  u32 *put_out, u32 *get_out)
+{
+	static u8 *buf;
+	u64 top = gsp_sema_gr_scan_top;
+	u64 ring_pa = 0;
+	u64 start = ch->gr_ram_userd_cursor;
+	u64 end, gpa;
+	u32 entries = ch->gpfifo_entries;
+	u64 best = 0;
+	u32 best_put = 0, best_get = 0;
+	bool best_pending = false;
+	int ring_ap = SVC_APERTURE_SYS;
+	int i;
+	unsigned long flags;
+
+	if (top > SVC_GR_SCAN_TOP_MAX)
+		top = SVC_GR_SCAN_TOP_MAX;
+	if (top < SEMA_PAGE_SIZE || !entries)
+		return false;
+	if (start >= top)
+		start = 0;
+	end = start + SVC_GR_RAM_SLICE;
+	if (end > top)
+		end = top;
+	if (!buf) {
+		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+		if (!buf)
+			return false;
+	}
+	(void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap);
+	for (gpa = start; gpa + SEMA_PAGE_SIZE <= end;
+	     gpa += SVC_SCAN_CHUNK) {
+		u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa);
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, chunk))
+			continue;
+		for (o = 0; o + 0x90 <= chunk; o += 0x100) {
+			u32 hp, hg, k;
+			bool pending, ok = true;
+
+			memcpy(&hg, buf + o + SVC_USERD_GP_GET, 4);
+			memcpy(&hp, buf + o + SVC_USERD_GP_PUT, 4);
+			/*
+			 * r914: the control block lives in a near-empty page
+			 * (only the put/get pair and a few engine-owned words
+			 * are written - the measured X-channel FB block and
+			 * the GL client's live sysmem block both leave the
+			 * leading 0x80 bytes zero). The shape gate alone is
+			 * far too weak: a low-RAM page at pa 0x7c000 carrying
+			 * coincidental small numbers at +0x88/+0x8c was
+			 * latched on r913 (the pass then read its put
+			 * forever and never advanced). Also accept a
+			 * free-running put (measured: the GL client's put
+			 * counts 0..3 with entries=2, so the old
+			 * put <= entries bound would reject the live block).
+			 */
+			if (!hp || hp > 64 || hg > hp)
+				continue;
+			{
+				u32 z = 0, zi;
+
+				for (zi = 0; zi < 0x80; zi++)
+					if (!buf[o + zi])
+						z++;
+				if (z < 0x60)
+					continue;
+			}
+			/*
+			 * Never adopt the ring page itself: the ring VA's
+			 * own page holds GPFIFO entries and ring metadata,
+			 * and a latch there would make the pump's GP_GET
+			 * write-back corrupt it.
+			 */
+			if (ring_pa && (gpa + o) ==
+			    (ring_pa & ~(u64)(SEMA_PAGE_SIZE - 1)))
+				continue;
+			for (k = 0; k < hp && k < 8; k++) {
+				u8 e[8];
+				u64 eva;
+				u32 elen, a0, a1;
+				/*
+				 * r914: entry slot j of a free-running put is
+				 * j % entries, not j.
+				 */
+				u32 slot = k % entries;
+
+				if (svc_read_va(vgpu,
+						ch->gpfifo_va +
+						(u64)slot * SVC_GPFIFO_ENTRY_SIZE,
+						e, sizeof(e))) {
+					ok = false;
+					break;
+				}
+				memcpy(&a0, e, 4);
+				memcpy(&a1, e + 4, 4);
+				eva = ((u64)(a1 & 0xff) << 32) |
+				      (u64)(a0 & 0xfffffffc);
+				elen = ((a1 >> 10) & 0x1fffff) << 2;
+				if (!eva ||
+				    eva + SVC_GR_RAM_WIN / 2 <
+					ch->gpfifo_va ||
+				    eva >= ch->gpfifo_va +
+					SVC_GR_RAM_WIN / 2 ||
+				    elen < 4 || elen > 0x100000) {
+					ok = false;
+					break;
+				}
+			}
+			if (!ok)
+				continue;
+			pending = hg < hp;
+			/* First pending candidate wins; a pending
+			 * candidate replaces a non-pending one. */
+			if (best && (best_pending || !pending))
+				continue;
+			best = gpa + o;
+			best_put = hp;
+			best_get = hg;
+			best_pending = pending;
+			if (pending)
+				break;
+		}
+		if (best_pending)
+			break;
+	}
+	ch->gr_ram_userd_cursor = (end >= top) ? 0 : end;
+	ch->gr_ram_userd_passes++;
+	if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged) {
+		ch->gr_ram_userd_logged = true;
+		pr_info("sema-chan: GR USERD h=0x%x rank=%u RAM scan found no candidate in %u slice(s) (top 0x%llx)\n",
+			ch->h_object, ch->gr_index, ch->gr_ram_userd_passes,
+			top);
+	}
+	/*
+	 * Persist the cursor immediately: the stalled-userd pump path
+	 * `continue`s past the end-of-pass write-back (the same reason
+	 * svc_gr_ram_ring_scan persists its cursor).
+	 */
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+		    svc.chan[i].h_object == ch->h_object) {
+			svc.chan[i].gr_ram_userd_cursor =
+				ch->gr_ram_userd_cursor;
+			svc.chan[i].gr_ram_userd_passes =
+				ch->gr_ram_userd_passes;
+			svc.chan[i].gr_ram_userd_logged =
+				ch->gr_ram_userd_logged;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+	if (!best)
+		return false;
+	*pa_out = best;
+	*put_out = best_put;
+	*get_out = best_get;
+	return true;
+}
+
+/*
  * r705h: host-triggered pdb discovery. The automatic latch
  * (svc_vas_candidates_verify + svc_xvas_pdb_discover) only runs from the RAM
  * ring scan, which sweeps [0,6GB) before the client's pushes exist and then
@@ -18657,6 +18896,25 @@ static void svc_pb_pair_gr(struct svc_channel *ch)
 	ring_va = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1);
 	want = SVC_PB_GR_SLOT_BASE | ch->gr_index;
 
+	/*
+	 * r914: replace this rank's previous pair instead of appending.
+	 * svc_pb_win_lookup returns the FIRST matching slot entry, so a
+	 * stale earlier pair (previous attempt's ring PA) would keep
+	 * winning over the refreshed one; svc_chan_rearm_rank re-arms this
+	 * pairing per (re-)registration for exactly that reason.
+	 */
+	{
+		u32 i;
+
+		for (i = 0; i < svc_pb_n; i++) {
+			if (svc_pb[i].slot == want) {
+				svc_pb[i] = svc_pb[svc_pb_n - 1];
+				svc_pb_n--;
+				i--;
+			}
+		}
+	}
+
 	/* Lower half: [ring - 1 MB, ring) -> [ring_pa - 1 MB, ring_pa). */
 	if (svc_pb_n < SVC_PB_MAX && ring_va >= SVC_GR_RAM_WIN / 2 &&
 	    ring_pa >= SVC_GR_RAM_WIN / 2) {
@@ -30745,6 +31003,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor;
 				svc.chan[k].gr_ram_passes = ch->gr_ram_passes;
 				svc.chan[k].gr_ram_logged = ch->gr_ram_logged;
+				/* r912: the guest-RAM USERD scan cursor (also
+				 * persisted immediately inside the scan - the
+				 * stalled path skips this write-back). */
+				svc.chan[k].gr_ram_userd_cursor =
+					ch->gr_ram_userd_cursor;
+				svc.chan[k].gr_ram_userd_passes =
+					ch->gr_ram_userd_passes;
+				svc.chan[k].gr_ram_userd_logged =
+					ch->gr_ram_userd_logged;
 				/* SESSION 149 R15: M1.1b keep-alive state. */
 				svc.chan[k].gr_ram_kick = ch->gr_ram_kick;
 				svc.chan[k].gr_ram_refreshes =
diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py
index f2deacd..de2e706 100644
--- a/tools/guest-diag/pin_gr_stream.py
+++ b/tools/guest-diag/pin_gr_stream.py
@@ -242,9 +242,17 @@ def _demand_loop():
                 r"(?:stalled VA #\d+[:=]\s*"
                 r"|pending release starved va="
                 r"|GR ring h=0x[0-9a-f]+ rank=\d+[^\n]*?\(va=)"
-                r"(0x[0-9a-f]+)",
+                r"(0x[0-9a-f]+)"
+                # r914: an unreadable GR push names its pushbuffer VA -
+                # the GL client's push page is a kernel allocation the
+                # userspace pagemap cannot cover, so the guest's content
+                # resolver is the only source (same class as the ring VA
+                # demand above). Harvest it so the guest resolves and
+                # injects the page.
+                r"|sema-pushdec: h=0x[0-9a-f]+ va=(0x[0-9a-f]+)"
+                r"[^\n]*unreadable=1",
                 (r.stdout or "")[-200000:]):
-            va = int(m.group(1), 16)
+            va = int(m.group(1) or m.group(2), 16)
             track.setdefault(va, [now, 0.0])
         for va in list(track):
             first, last = track[va]