r496b the X/GL channel bring-up: sysmem ring discovery by content + VRAM USERD shape scan

With the GR classification fixed the pump reaches the X driver's 3D channel
for the first time, but both of its structures are outside every existing
translation source: the CPU RM maps the pushbuffer's NV01_MEMORY_SYSTEM
surface locally (no SET_PAGE_DIRECTORY, no rm_mem record carrying the ring
VA - the targeted resolve scans all of guest RAM and finds nothing), and the
driver's USERD GP_PUT write lands in the emulated VRAM at a BAR1-window
offset (observed: fb 0x80000) that is neither the registered memdesc base
(0x10000, pool-relative) nor ring-derived.

Two content scans close the gap, both validated so a wrong hit costs a
failed probe, not a bad pin:
- svc_gr_ram_ring_scan: the driver's kickoff writes GPFIFO entries whose
  pushbuffer VAs sit just BELOW the ring VA in the same allocation (main
  pushbuffer first, GPFIFO after - nvidia-push-priv.h). A page whose first
  two 8-byte entries both name such VAs with sane lengths is the ring; the
  ring page is pinned validated and the 2 MB allocation window as guesses
  (the ring entry and pushbuffer reads resolve through it). Sliced 512 MB
  per pass, cursor-resumed, 16 slices max.
- svc_userd_fb_scan (svc_chan_userd GR fallback): scan the VRAM for the
  USERD shape (GP_PUT at +0x8c in [1..entries], GP_GET at +0x88 <= put) and
  validate each hit against the ring: every entry the put claims must
  decode to a pushbuffer VA in the ring's allocation window. A foreign
  channel's USERD cannot pass that test. 8 bounded retries (the kickoff
  lands a few passes after registration).

1 files changed, 254 insertions, 2 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b328eea..04bffc5 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -160,6 +160,15 @@ struct sema_map_entry {
 #define SVC_USERD_GP_GET	0x88
 #define SVC_USERD_GP_PUT	0x8c
 
+/*
+ * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct
+ * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan
+ * slice per service pass so the service thread stays responsive.
+ */
+#define SVC_GR_RAM_SLICE	(512ULL << 20)	/* guest RAM per pass */
+#define SVC_GR_RAM_PASS_MAX	16		/* slices before giving up */
+#define SVC_GR_RAM_WIN		(2ULL << 20)	/* allocation window pinned */
+
 struct svc_channel {
 	bool	valid;
 	u32	session_gen;	/* m176: session generation this entry was
@@ -420,6 +429,27 @@ struct svc_channel {
 	u64	gr_nocap_va;	/* push VA the window was opened for */
 	unsigned long gr_nocap_since;	/* absolute first-seen stamp */
 	/*
+	 * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in
+	 * guest RAM. The CPU RM maps sysmem objects locally (no
+	 * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so
+	 * svc_lookup_trusted cannot name the ring page; the driver's kickoff
+	 * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan
+	 * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the
+	 * service thread stays responsive.
+	 */
+	u64	gr_ram_cursor;	/* resume point of the content scan */
+	u8	gr_ram_passes;	/* slices walked so far */
+	bool	gr_ram_logged;	/* one discovery line per channel */
+	/*
+	 * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the
+	 * registered/ring-derived candidates all fail; the scan validates the
+	 * hit against the ring entries the put claims, so a foreign channel's
+	 * USERD cannot be adopted. Bounded retries: the driver kicks off a
+	 * few passes after channel registration, so a scan before the first
+	 * put must not retire the discovery.
+	 */
+	u8	userd_scans;
+	/*
 	 * r471: the completion-counter publish must never advance the
 	 * guest's word past a push whose content was still unwritten when
 	 * the entry was consumed (the streams_events staging-reuse
@@ -12999,6 +13029,77 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			return 0;
 		}
 	}
+
+	/*
+	 * r496b: no candidate matched. For a GR channel whose ring is known
+	 * (translated and pinned - the sysmem content scan or the VRAM
+	 * conventions), the USERD may still be elsewhere: the X/GL driver
+	 * allocates its USERD in VRAM and the CPU RM maps it through a BAR1
+	 * window, so the driver's GP_PUT write lands at a BAR1 offset that is
+	 * neither the registered memdesc base nor ring-derived. Scan the
+	 * emulated VRAM for the USERD shape and validate each hit against the
+	 * ring: the put it claims must correspond to entries whose pushbuffer
+	 * VAs all sit in the ring's allocation window. A foreign channel's
+	 * USERD cannot pass that test. One scan per channel (the result
+	 * persists through the pump's write-back), logged on latch.
+	 */
+	if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) {
+		u64 off, lim;
+		u8 *base = vgpu->bars[1].vaddr;
+
+		ch->userd_scans++;
+		lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
+		if (base && lim) {
+			for (off = 0; off + 0xa0 <= lim; off += 0x200) {
+				u32 hp, hg;
+				bool ok = true;
+				u32 k;
+
+				memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4);
+				memcpy(&hg, base + off + SVC_USERD_GP_GET, 4);
+				if (!hp || hp > ch->gpfifo_entries || hg > hp)
+					continue;
+				/* Validate: every claimed entry decodes to a
+				 * pushbuffer VA in the ring's window. */
+				for (k = 0; k < hp && k < 64; k++) {
+					u8 e[8];
+					u64 eva;
+					u32 elen, a0, a1;
+
+					if (svc_read_va(vgpu,
+							ch->gpfifo_va +
+							(u64)k * 8,
+							e, sizeof(e))) {
+						ok = false;
+						break;
+					}
+					memcpy(&a0, e, 4);
+					memcpy(&a1, e + 4, 4);
+					eva = ((u64)(a1 & 0xff) << 32) |
+					      (u64)(a0 & 0xfffffffc);
+					elen = ((a1 >> 10) & 0x1fffff) << 2;
+					if (!eva ||
+					    eva >= ch->gpfifo_va ||
+					    eva + SVC_GR_RAM_WIN / 2 <
+						ch->gpfifo_va ||
+					    elen < 4 || elen > 0x100000) {
+						ok = false;
+						break;
+					}
+				}
+				if (!ok)
+					continue;
+				ch->userd_pa = off;
+				ch->userd_as = SVC_APERTURE_FB;
+				ch->userd_ok = true;
+				*put = hp;
+				*get = hg;
+				pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u, ring-validated)\n",
+					ch->h_object, ch->gr_index, off, hp, hg);
+				return 0;
+			}
+		}
+	}
 	return -ENOENT;
 }
 
@@ -14034,6 +14135,72 @@ unsigned gsp_sema_push_hold_ms = 2500;
 
 static unsigned long svc_gr_ring_tried;	/* ranks that got a diagnostic line */
 
+static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
+				 struct svc_channel *ch, u64 *pa_out)
+{
+	static u8 *buf;
+	u64 start = ch->gr_ram_cursor;
+	u64 end = start + SVC_GR_RAM_SLICE;
+	u64 gpa;
+
+	if (end > SEMA_PA_MAX)
+		end = SEMA_PA_MAX;
+	if (!buf) {
+		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+		if (!buf)
+			return false;
+	}
+	for (gpa = start; gpa + SEMA_PAGE_SIZE <= end;
+	     gpa += SVC_SCAN_CHUNK) {
+		u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa);
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, chunk))
+			continue;
+		for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) {
+			u32 e0, e1, f0, f1;
+			u64 va0, va1;
+			u32 len0, len1;
+
+			memcpy(&e0, buf + o, 4);
+			memcpy(&e1, buf + o + 4, 4);
+			va0 = ((u64)(e1 & 0xff) << 32) |
+			      (u64)(e0 & 0xfffffffc);
+			len0 = ((e1 >> 10) & 0x1fffff) << 2;
+			/* entry0: main pushbuffer, just below the ring */
+			if (va0 >= ch->gpfifo_va ||
+			    va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va)
+				continue;
+			if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI)
+				continue;
+			if (len0 < 4 || len0 > 0x100000)
+				continue;
+			memcpy(&f0, buf + o + 8, 4);
+			memcpy(&f1, buf + o + 12, 4);
+			va1 = ((u64)(f1 & 0xff) << 32) |
+			      (u64)(f0 & 0xfffffffc);
+			len1 = ((f1 >> 10) & 0x1fffff) << 2;
+			/* entry1: progress tracker, above entry0 */
+			if (va1 <= va0 ||
+			    va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2)
+				continue;
+			if (len1 < 4 || len1 > 0x100000)
+				continue;
+			*pa_out = gpa + o;
+			return true;
+		}
+	}
+	ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
+	ch->gr_ram_passes++;
+	if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) {
+		ch->gr_ram_logged = true;
+		pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n",
+			ch->h_object, ch->gr_index, ch->gr_ram_passes,
+			ch->gpfifo_va);
+	}
+	return false;
+}
+
 static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 				struct svc_channel *ch)
 {
@@ -14310,6 +14477,85 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 			pr_info("sema-chan:   fbs scan done h=0x%x hits=%d (lim 0x%llx)\n",
 				ch->h_object, hits, lim);
 		}
+
+		/*
+		 * r495 diagnostic (X/GL channel): the X driver's GR channel
+		 * registers its USERD at a memdesc base that may be an FB
+		 * offset or a BAR1 window. The driver's kickoff writes GP_PUT
+		 * through its CPU mapping of that object; if the write landed
+		 * in the emulated FB shadow, scanning for the USERD shape
+		 * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90,
+		 * both nonzero somewhere in the 0x200 block) names the real
+		 * offset. One-shot per channel, diagnostic only.
+		 */
+		if (gsp_sema_dbg && vgpu->bars[1].vaddr &&
+		    svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) {
+			u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
+			u64 off;
+			int hits = 0;
+
+			for (off = 0; off + 0xa0 <= lim && hits < 8;
+			     off += 0x200) {
+				u32 hp, hg;
+
+				memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4);
+				memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4);
+				if (!hp || hp > ch->gpfifo_entries || hg > hp)
+					continue;
+				pr_info("sema-chan:   userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n",
+					off, hp, hg,
+					*(u32 *)(vgpu->bars[1].vaddr + off + 0x80),
+					*(u32 *)(vgpu->bars[1].vaddr + off + 0x84),
+					*(u32 *)(vgpu->bars[1].vaddr + off + 0x88),
+					*(u32 *)(vgpu->bars[1].vaddr + off + 0x94));
+				hits++;
+			}
+			pr_info("sema-chan:   userd scan done h=0x%x hits=%d (lim 0x%llx)\n",
+				ch->h_object, hits, lim);
+		}
+	}
+
+	/*
+	 * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor
+	 * field comment). The driver's kickoff writes GPFIFO entries whose
+	 * pushbuffer VAs sit just BELOW the ring VA in the same allocation
+	 * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0
+	 * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to
+	 * ring_va + 0x100000 (the progress tracker). A page whose first two
+	 * 8-byte words both name such VAs with sane lengths is the ring; pin
+	 * it (validated) and its allocation window (guesses) so the ring
+	 * entry and pushbuffer reads resolve. Sliced so the service thread
+	 * stays responsive; gives up per channel after
+	 * SVC_GR_RAM_PASS_MAX slices.
+	 */
+	if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX &&
+	    !test_bit(ch->gr_index, &svc_gr_ring_done)) {
+		u64 hit_pa = 0;
+
+		if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) {
+			u64 win_va = ch->gpfifo_va - SVC_GR_RAM_WIN / 2;
+			u64 win_pa = hit_pa - SVC_GR_RAM_WIN / 2;
+			u64 off;
+
+			svc_pin_store(ch->gpfifo_va, hit_pa,
+				      SVC_APERTURE_SYS);
+			svc_lat_store(ch->gpfifo_va, hit_pa,
+				      SVC_APERTURE_SYS);
+			for (off = 0; off < SVC_GR_RAM_WIN;
+			     off += SEMA_PAGE_SIZE) {
+				svc_pin_store_win(win_va + off, win_pa + off,
+						  SVC_APERTURE_SYS);
+				svc_lat_store(win_va + off, win_pa + off,
+					      SVC_APERTURE_SYS);
+			}
+			ch->gr_ram_logged = true;
+			set_bit(ch->gr_index, &svc_gr_ring_done);
+			svc_ring_n++;
+			pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n",
+				ch->h_object, ch->gr_index, ch->gpfifo_va,
+				hit_pa);
+			return;
+		}
 	}
 }
 
@@ -14317,8 +14563,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
 			     struct svc_channel *ch)
 {
 	u64 pa = 0, va = 0;
-	int ap = SVC_APERTURE_SYS;
-	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	int ap = SVC_APERTURE_SYS;	u8 e[SVC_GPFIFO_ENTRY_SIZE];
 	u32 e0, e1, len;
 
 	if (!ch->engine_type) {
@@ -22454,6 +22699,13 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				svc.chan[k].userd_pa = ch->userd_pa;
 				svc.chan[k].userd_as = ch->userd_as;
 				svc.chan[k].userd_ok = ch->userd_ok;
+				/* r496b: persist the RAM-scan cursor and the
+				 * one-shot USERD-scan latch so they survive
+				 * the per-pass snapshot copy. */
+				svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor;
+				svc.chan[k].gr_ram_passes = ch->gr_ram_passes;
+				svc.chan[k].gr_ram_logged = ch->gr_ram_logged;
+				svc.chan[k].userd_scans = ch->userd_scans;
 				svc.chan[k].logged = ch->logged;
 				svc.chan[k].logged_expected = ch->logged_expected;
 				svc.chan[k].notified = ch->notified;