vgsp: GL ring scan fixed (ring-own-offset entries, either-side window, va1==va0; scan top param) + ring VA demand regex (SESSION 153)

1 files changed, 96 insertions, 22 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 4674987..812cfea 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -197,6 +197,43 @@ struct sema_map_entry {
 #define SVC_GR_RAM_REFRESH_MS	10000		/* minimum spacing between them */
 #define SVC_GR_RAM_WIN		(2ULL << 20)	/* allocation window pinned */
 
+/*
+ * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM
+ * can extend above SEMA_PA_MAX: the 8 GiB guest's e820 top is
+ * 0x280000000, and the GL client channel's GPFIFO ring page was measured
+ * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there) -
+ * above the old 6 GB cap, so the content scan could never reach it. Only
+ * the ring scan uses this wider top; every other table/bound in the
+ * service stays sized on SEMA_PA_MAX. Reads past mapped RAM (the
+ * 0x80000000-0x100000000 PCI hole, or a smaller guest) fail fast, so a
+ * too-large top costs a few failed probes per sweep. The module parameter
+ * is the per-cycle guard: shrink it to the boot's real RAM top.
+ */
+unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL;
+module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644);
+#define SVC_GR_SCAN_TOP_MAX	0x400000000ULL	/* 16 GiB sanity clamp */
+
+/*
+ * Ring-scan slice budget: two full sweeps at the configured top, with the
+ * original 24-slice floor and a u8-safe cap (gr_ram_passes).
+ */
+static u32 svc_gr_ram_pass_budget(void)
+{
+	u64 top = gsp_sema_gr_scan_top;
+	u32 slices;
+
+	if (top > SVC_GR_SCAN_TOP_MAX)
+		top = SVC_GR_SCAN_TOP_MAX;
+	if (top <= SVC_GR_RAM_SLICE)
+		return SVC_GR_RAM_PASS_MAX;
+	slices = (u32)(top / SVC_GR_RAM_SLICE) * 2;
+	if (slices < SVC_GR_RAM_PASS_MAX)
+		slices = SVC_GR_RAM_PASS_MAX;
+	if (slices > 250)
+		slices = 250;
+	return slices;
+}
+
 struct svc_channel {
 	bool	valid;
 	u32	session_gen;	/* m176: session generation this entry was
@@ -17573,14 +17610,31 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 				 struct svc_channel *ch, u64 *pa_out)
 {
 	static u8 *buf;
+	u64 top = gsp_sema_gr_scan_top;
 	u64 start = ch->gr_ram_cursor;
-	u64 end = start + SVC_GR_RAM_SLICE;
+	u64 end;
 	u64 gpa;
+	/*
+	 * r908: the GPFIFO entries start AT the registered ring VA, so their
+	 * in-page offset is the ring VA's own offset (X server ring
+	 * 0x120090000 -> offset 0; RC watchdog 0x120000400 -> 0x400; the GL
+	 * client 0x1203cfff0 -> 0xff0, measured). The scan used to test only
+	 * page offset 0, so any ring whose VA is not page-aligned could never
+	 * match no matter how good the entry contents were.
+	 */
+	u32 eoff = (u32)(ch->gpfifo_va & (SEMA_PAGE_SIZE - 1));
 	int i;
 	unsigned long flags;
 
-	if (end > SEMA_PA_MAX)
-		end = SEMA_PA_MAX;
+	if (top > SVC_GR_SCAN_TOP_MAX)
+		top = SVC_GR_SCAN_TOP_MAX;
+	if (top < SEMA_PAGE_SIZE)
+		return false;
+	if (start >= top)
+		start = 0;
+	end = start + SVC_GR_RAM_SLICE;
+	if (end > top)
+		end = top;
 	if (!buf) {
 		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
 		if (!buf)
@@ -17593,31 +17647,46 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 
 		if (svc_read(vgpu, gpa, buf, chunk))
 			continue;
-		for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) {
+		for (o = 0; o + eoff + 2 * SVC_GPFIFO_ENTRY_SIZE <= chunk;
+		     o += SEMA_PAGE_SIZE) {
 			u32 e0, e1, f0, f1;
 			u64 va0, va1;
 			u32 len0, len1;
 
-			memcpy(&e0, buf + o, 4);
-			memcpy(&e1, buf + o + 4, 4);
+			memcpy(&e0, buf + o + eoff, 4);
+			memcpy(&e1, buf + o + eoff + 4, 4);
 			va0 = ((u64)(e1 & 0xff) << 32) |
 			      (u64)(e0 & 0xfffffffc);
 			len0 = ((e1 >> 10) & 0x1fffff) << 2;
-			/* entry0: main pushbuffer, just below the ring */
-			if (va0 >= ch->gpfifo_va ||
-			    va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va)
+			/*
+			 * entry0: a pushbuffer VA in the ring's allocation
+			 * window - just below the ring for the long-standing
+			 * convention (X ring: 0x120010000 for ring
+			 * 0x120090000), or just above it (r908: the GL
+			 * client's PROMOTE_CTX ordering shows the main
+			 * pushbuffer above the ring on this boot; accept
+			 * both sides, still bounded to +/-1 MB of the ring).
+			 */
+			if (va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va ||
+			    va0 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2)
 				continue;
 			if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI)
 				continue;
 			if (len0 < 4 || len0 > 0x100000)
 				continue;
-			memcpy(&f0, buf + o + 8, 4);
-			memcpy(&f1, buf + o + 12, 4);
+			memcpy(&f0, buf + o + eoff + 8, 4);
+			memcpy(&f1, buf + o + eoff + 12, 4);
 			va1 = ((u64)(f1 & 0xff) << 32) |
 			      (u64)(f0 & 0xfffffffc);
 			len1 = ((f1 >> 10) & 0x1fffff) << 2;
-			/* entry1: progress tracker, above entry0 */
-			if (va1 <= va0 ||
+			/*
+			 * entry1: the progress tracker above entry0 - or the
+			 * SAME push submitted again (r908 measured on the GL
+			 * client's ring: {push=0x1203c0000,len=64} then
+			 * {push=0x1203c0000,len=32}). Never below entry0;
+			 * still bounded by the window above the ring.
+			 */
+			if (va1 < va0 ||
 			    va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2)
 				continue;
 			if (len1 < 4 || len1 > 0x100000)
@@ -17626,7 +17695,7 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 			return true;
 		}
 	}
-	ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
+	ch->gr_ram_cursor = (end >= top) ? 0 : end;
 	ch->gr_ram_passes++;
 	if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) {
 		ch->gr_ram_logged = true;
@@ -18403,12 +18472,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 	 * pushbuffer VAs sit just BELOW the ring VA in the same allocation
 	 * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0
 	 * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to
-	 * ring_va + 0x100000 (the progress tracker). A page whose first two
-	 * 8-byte words both name such VAs with sane lengths is the ring; pin
-	 * it (validated) and its allocation window (guesses) so the ring
-	 * entry and pushbuffer reads resolve. Sliced so the service thread
-	 * stays responsive; gives up per channel after
-	 * SVC_GR_RAM_PASS_MAX slices.
+	 * ring_va + 0x100000 (the progress tracker). r908 relaxations,
+	 * measured on the GL client ring (va=0x1203cfff0, gpa 0x231e5d000):
+	 * the entry pair is tested at the ring VA's own in-page offset (the
+	 * entries start AT the registered VA), entry0 may sit either side of
+	 * the ring within the window, and entry1 may repeat entry0's VA
+	 * (that client queues {push,64} then {push,32}). A page whose entry
+	 * pair both name such VAs with sane lengths is the ring; pin it
+	 * (validated) and its allocation window (guesses) so the ring entry
+	 * and pushbuffer reads resolve. Sliced so the service thread stays
+	 * responsive; gives up per channel after svc_gr_ram_pass_budget()
+	 * slices (two sweeps of sema_gr_scan_top).
 	 *
 	 * SESSION 147 (r840 evidence): the scan must NOT run for the kernel
 	 * RC-watchdog channel (h=0x31415900). Its ring is a plain kernel
@@ -18426,7 +18500,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 	 * latch.
 	 */
 	if (ch->h_object != 0x31415900u &&
-	    ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX &&
+	    ch->gr_ram_passes < svc_gr_ram_pass_budget() &&
 	    !test_bit(ch->gr_index, &svc_gr_ring_done)) {
 		u64 hit_pa = 0;
 
@@ -28062,7 +28136,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		if (!ch->engine_type && ch->h_object != 0x31415900u &&
 		    ch->gr_index < 32 &&
 		    !test_bit(ch->gr_index, &svc_gr_ring_done) &&
-		    ch->gr_ram_passes >= SVC_GR_RAM_PASS_MAX &&
+		    ch->gr_ram_passes >= svc_gr_ram_pass_budget() &&
 		    ch->gr_ram_refreshes < SVC_GR_RAM_REFRESH_MAX &&
 		    time_after_eq(jiffies, ch->gr_ram_refresh_at) &&
 		    ch->gr_ram_kick != put) {