vgsp-sema: discover direct-GL CE userd pages; complete kick slots; post-reboot diagnostics

The direct-GL GL client's CE channels (beef0101/0102) declare a
pool-relative FB control page (0x244f800 / 0x244f400 in r1656) while their
GPFIFO ring is mapped in guest RAM, and their ring publishes from slot 2
(entry 0 stays zero). The entry-0 layout gate could never fire, the BAR0
fallback rejects offsets past its 16 MB shadow, and the GR-only VRAM shape
scan never ran for them, so the channels retired with submissions=0 and
the client's CE releases were never applied.

r1661/r1663 evidence: enabling the VRAM shape scan for the beef CE family
(hole-tolerant entry validation, newest-slot required, a GET bounded to
one ring) finds the real control page (fb 0xa00000, put=1) and the CE push
at 0x40e0000 decodes; beef0101 now retires submissions=1 with its
semaphore releases applied.

Also:
- resolve the non-pool direct-GL semaphore window (0x4000000..0x4300000)
  through every published VAS translation (the general resolver can name
  a stale page while the client polls the beef0300 walk page);
- author 0xffffffff in the beef channel's 128 kick-slot records (the
  glcore 570.144 kick wait polls exactly that payload; r1662 writes are
  visible in the guest page, but this is not yet the X-server wall);
- one-shot nonzero-word dump of the beef0100 push (r1663 proved the GR
  init push's host semaphore is a true 32-bit VA 0x4238000 payload 1 with
  hi=0 in v1, so that release is correct);
- name CE/GR in the VRAM-scan adoption line.

Next wall (r1663): the X server spins in _nv015102rm/os_schedule (66 %) and
_nv018351rm (30 %) after the GL client's GLX request; GZDoom blocks in
_xcb_wait_for_reply. The display UPDATE completion event was posted and
re-posted at 12:12:20, so this is the flip/GLX-sync path, not the lost
UPDATE event.

1 files changed, 224 insertions, 6 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index f7a5c7a..f758218 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -15802,6 +15802,7 @@ static u64 svc_pushraw_last_va;
 static u32 svc_pushraw_last_len;
 static u32 svc_pushraw_n;
 static u32 svc_pushraw_small_n;
+static u32 svc_pushraw_nz;
 
 /*
  * SESSION 153-CONTINUED-52: MME-family method census for the beef (GL 3D
@@ -18569,6 +18570,28 @@ static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va,
 	}
 	pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n",
 		h_object, push_va, off, len);
+	/*
+	 * r1662 diag: the direct-GL GR init push reads a zero 640-byte head
+	 * and the semantic decoder still finds its release; dump the real
+	 * nonzero words once per handle so the notification method (if any)
+	 * and the slot address can be identified.
+	 */
+	if (h_object == 0xbeef0100u && svc_pushraw_nz < 96) {
+		u32 p;
+
+		for (p = 0; p + 4 <= len; p += 4) {
+			u32 w;
+
+			memcpy(&w, buf + p, 4);
+			if (!w)
+				continue;
+			if (svc_pushraw_nz >= 96)
+				break;
+			svc_pushraw_nz++;
+			pr_info("sema-pushraw: NZ h=0x%x off=0x%x v=%08x\n",
+				h_object, p, w);
+		}
+	}
 }
 
 /*
@@ -24735,6 +24758,96 @@ static void svc_kick_capture_arm(struct nv_vgpu_device *vgpu,
 			ch->h_object, ch->notify2d_va);
 }
 
+/*
+ * r1659: direct-GL CE USERD adoption. The beef0101/0102 channels declare a
+ * real FB control page (userdMem.base + userdOffset: 0x244f800 / 0x244f400)
+ * while their GPFIFO ring is mapped in guest RAM. svc_chan_userd only
+ * probed ring+0x10000 in the ring's aperture and the 16 MB BAR0 shadow, so
+ * the declared page was never read; their ring also starts at slot 2
+ * (entry 0 stays zero), which the entry-0 layout gate can never accept.
+ * Read the declared page through the FB shadow and adopt it when its PUT is
+ * plausible and the newest claimed slot decodes into this channel's low-VA
+ * push window. Holes below the live PUT are allowed (the client's CE ring
+ * has unwritten slots between its publishes).
+ */
+static bool svc_ce_declared_userd(struct nv_vgpu_device *vgpu,
+				  struct svc_channel *ch, u32 *put, u32 *get)
+{
+	u32 p = 0, g = 0, k, slot, valid = 0, newest, newest_ok = 0;
+	u32 scan_end;
+
+	if (!ch->userd_pa || !ch->gpfifo_entries)
+		return false;
+	if (svc_userd_probe_ex(vgpu, ch->userd_pa, ch->userd_as,
+			       ch->gpfifo_entries, &p, &g, true))
+		return false;	/* unreadable/idle/incoherent: retry later */
+	if (!p)
+		return false;
+	newest = (p - 1) % ch->gpfifo_entries;
+	/*
+	 * Validate the newest claimed slot first: that is the entry the
+	 * client is waiting on, and a stale/foreign control page cannot
+	 * name a push in this channel's own ring window. Allow a few holes
+	 * below the newest (the live CE ring publishes at slots 2/4/5).
+	 */
+	for (k = 0; k < 8 && k < p && !newest_ok; k++) {
+		u8 e[8];
+		u32 e0, e1, len;
+		u64 eva;
+
+		slot = (p - 1 - k) % ch->gpfifo_entries;
+		if (svc_read_va(vgpu, ch->gpfifo_va +
+				(u64)slot * SVC_GPFIFO_ENTRY_SIZE,
+				e, sizeof(e)))
+			continue;
+		memcpy(&e0, e, 4);
+		memcpy(&e1, e + 4, 4);
+		eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+		len = ((e1 >> 10) & 0x1fffff) << 2;
+		if (!eva && !len)
+			continue;	/* unwritten hole */
+		if (!eva || len < 4 || len > 0x100000 ||
+		    !svc_gr_entry_win_ok(ch->gpfifo_va, eva))
+			return false;
+		newest_ok = 1;
+	}
+	if (!newest_ok)
+		return false;
+	scan_end = p < 64 ? p : 64;
+	for (k = 0; k < scan_end; k++) {
+		u8 e[8];
+		u32 e0, e1, len;
+		u64 eva;
+
+		slot = k % ch->gpfifo_entries;
+		if (svc_read_va(vgpu, ch->gpfifo_va +
+				(u64)slot * SVC_GPFIFO_ENTRY_SIZE,
+				e, sizeof(e)))
+			continue;
+		memcpy(&e0, e, 4);
+		memcpy(&e1, e + 4, 4);
+		eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+		len = ((e1 >> 10) & 0x1fffff) << 2;
+		if (!eva && !len)
+			continue;	/* unwritten hole */
+		if (!eva || len < 4 || len > 0x100000 ||
+		    !svc_gr_entry_win_ok(ch->gpfifo_va, eva))
+			return false;
+		valid++;
+	}
+	if (!valid)
+		return false;
+	ch->userd_ok = true;
+	ch->userd_wait_fresh = p >= 1;
+	ch->last_get = 0;	/* live sequence starts at slot 0; holes skip */
+	*put = p;
+	*get = g;
+	pr_info("sema-chan: CE USERD h=0x%x eng=%u ADOPT declared pa=0x%llx as=%u put=%u get=%u (newest slot %u validated)\n",
+		ch->h_object, ch->engine_type, ch->userd_pa, ch->userd_as,
+		p, g, newest);
+	return true;
+}
+
 static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			  u64 ring_pa, int ring_ap, u32 *put, u32 *get, int rrc)
 {
@@ -24890,6 +25003,14 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	 */
 	if (ch->engine_type != 0) {
 		/*
+		 * r1659: the direct-GL CE channels declare their control
+		 * page in FB (0x244f800/0x244f400) and publish from slot 2,
+		 * so the entry-0 layout gate above never fires. Adopt the
+		 * declared page when the newest slot it names validates.
+		 */
+		if (svc_ce_declared_userd(vgpu, ch, put, get))
+			return 0;
+		/*
 		 * SESSION 153-CONT-34: before giving up on a CE channel, try
 		 * the registered offset read as a BAR0-shadow USERD - the GL
 		 * client's copy/2D channels declare the same FB offsets as its
@@ -24898,7 +25019,13 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 		if (svc_bar0_userd_try(vgpu, ch, put, get))
 			return 0;
 		svc_userd_sweep_sched(ch);
-		return -ENOENT;
+		/*
+		 * r1659: the direct-GL CE channels' real control page is
+		 * not the declared pool-relative offset; find it with the
+		 * same VRAM shape scan the GR path uses (now hole-tolerant
+		 * for CE). Jump past the GR-only candidate/pin paths.
+		 */
+		goto ce_fb_scan;
 	}
 
 	if (ring_pa) {
@@ -25122,7 +25249,10 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	 * keep retrying up to a wider cap; the client's own timeout still
 	 * bounds the channel's life.
 	 */
-	if (!ch->engine_type && ring_pa &&
+ce_fb_scan:
+	if (ring_pa &&
+	    (!ch->engine_type ||
+	     (ch->h_object & 0xfffff000u) == 0xbeef0000u) &&
 	    ch->fb_userd_scan_n < 512 &&
 	    (!ch->fb_userd_scan_at ||
 	     time_after_eq(jiffies,
@@ -25158,6 +25288,7 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			for (off = 0; off + 0xa0 <= lim; off += 0x200) {
 				u32 hp, hg;
 				bool ok = true;
+				bool newest_ok = false;
 				u32 k;
 				u64 dist;
 				bool pending;
@@ -25169,9 +25300,15 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 				memcpy(&hg, base + off + SVC_USERD_GP_GET, 4);
 				/* CONT-38: large GR kernel rings may carry an
 				 * uninitialised GPU-owned GP_GET; the PUT alone
-				 * gates there (see svc_userd_probe_ex). */
+				 * gates there (see svc_userd_probe_ex). r1659:
+				 * CE control pages also carry a GPU-owned GET,
+				 * but bound it to one ring so a garbage pair
+				 * (r1660: put=2880 get=188746560) cannot pass. */
 				if (!hp || hp > ch->gpfifo_entries ||
-				    (hg > hp && ch->gpfifo_entries <= 1024))
+				    (hg > hp &&
+				     (ch->engine_type ?
+				      hg > hp + ch->gpfifo_entries :
+				      ch->gpfifo_entries <= 1024)))
 					continue;
 				gate_n++;
 				/* Validate: every claimed entry decodes to a
@@ -25235,6 +25372,15 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 					 * kernel-VA family is accepted in its
 					 * own low window (CONT-38).
 					 */
+					/*
+					 * r1659: the direct-GL CE ring starts
+					 * at slot 2 (entry 0/1 stay zero until
+					 * the client's first publishes); an
+					 * unwritten hole is not a window
+					 * failure for a CE channel.
+					 */
+					if (!eva && !elen && ch->engine_type)
+						continue;
 					if (!eva || !svc_gr_entry_win_ok(
 							   ch->gpfifo_va, eva) ||
 					    elen < 4 || elen > 0x100000) {
@@ -25244,7 +25390,15 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 						rklen = elen;
 						break;
 					}
+					if (ch->engine_type &&
+					    k == (u32)((hp - 1) %
+						       ch->gpfifo_entries))
+						newest_ok = true;
 				}
+				/* r1659: a CE control page must account
+				 * for the slot the client is waiting on. */
+				if (ch->engine_type && !newest_ok)
+					ok = false;
 				if (!ok) {
 					/*
 					 * r820 diag: name why a scanned
@@ -25313,8 +25467,11 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 				 */
 				ch->gr_ram_userd_latch_at = 0;
 				ch->gr_ram_userd_confirmed = false;
-				pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n",
-					ch->h_object, ch->gr_index, best,
+				pr_info("sema-chan: %s USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n",
+					ch->engine_type ? "CE" : "GR",
+					ch->h_object,
+					ch->engine_type ? ch->ce_index
+							: ch->gr_index, best,
 					best_put, best_get,
 					best_pending ? "" : ", idle");
 				return 0;
@@ -26125,6 +26282,21 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
  */
 static u32 svc_client_slot_log_n;
 static unsigned long svc_client_slot_at;
+/*
+ * r1662: glcore's kick wait (570.144: wait_kick_complete 0xa03930, predicate
+ * 0x9fde70) polls a u32 == 0xffffffff at notify_va + 0x200 + seq*16 + 8.
+ * The client clears the record to 0 and writes its tag at
+ * notify_va + 8 + idx*4 before submitting; the GPU completion writes the
+ * payload. The emulation only cleared the u16 status below the notifier, so
+ * every kick sat out its deadline. Complete every tagged record (the client
+ * clears before reuse, so completing a stale one is harmless).
+ */
+static bool gsp_sema_kick_complete = true;
+module_param_named(sema_kick_complete, gsp_sema_kick_complete, bool, 0644);
+MODULE_PARM_DESC(sema_kick_complete,
+	"Author the GL client's kick notification payload 0xffffffff (default 1)");
+static unsigned long svc_kick_complete_at;
+static u32 svc_kick_complete_n;
 
 static void svc_client_slot_sweep(struct nv_vgpu_device *vgpu)
 {
@@ -26181,7 +26353,44 @@ static void svc_client_slot_sweep(struct nv_vgpu_device *vgpu)
 				pr_info("sema-chan: client-slot cleared(sweep) h=0x%x slot=%d pa=0x%llx n=%u\n",
 					tab[i].h, k, pa, svc_client_slot_log_n);
 		}
+		/*
+		 * r1662: complete the kick-slot records (payload at
+		 * notify+0x200+seq*16+8 == 0xffffffff). Rate-limited to
+		 * ~100 Hz; the client's wait slice is 1 ms but its overall
+		 * deadline is seconds, so a few passes of latency are fine.
+		 */
+		if (gsp_sema_kick_complete &&
+		    time_after_eq(jiffies, svc_kick_complete_at)) {
+			u32 done = 0xffffffffu;
+			int q;
+
+			for (q = 0; q < 120; q++) {
+				u32 tag = 0;
+
+				if (svc_read_ap(vgpu,
+						tab[i].base + 8 + (u64)q * 4,
+						tab[i].as, &tag, sizeof(tag)))
+					continue;
+				if (!tag)
+					continue;
+				if (svc_write_ap(vgpu,
+						 tab[i].base + 0x200 +
+						 (u64)q * SVC_NOTIFY_SIZE + 8,
+						 tab[i].as, &done,
+						 sizeof(done)))
+					continue;
+				svc_kick_complete_n++;
+				if (gsp_sema_dbg && svc_kick_complete_n <= 64)
+					pr_info("sema-chan: kick-slot completed h=0x%x seq=%d pa=0x%llx\n",
+						tab[i].h, q,
+						tab[i].base + 0x200 +
+						(u64)q * SVC_NOTIFY_SIZE + 8);
+			}
+		}
 	}
+	if (gsp_sema_kick_complete &&
+	    time_after_eq(jiffies, svc_kick_complete_at))
+		svc_kick_complete_at = jiffies + msecs_to_jiffies(10);
 }
 
 /*
@@ -40457,6 +40666,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		if (!ch->gpfifo_va || !ch->gpfifo_entries)
 			continue;
 
+		/*
+		 * r1659: every channel turn answers its own VAs first. The
+		 * GR block below also does this before its autoreg derive;
+		 * CE channels need it before USERD discovery so the declared
+		 * control page can be validated against this channel's ring
+		 * entries through this channel's VAS.
+		 */
+		svc_cur_vas = ch->h_vaspace;
+
 		svc_ring_autopin(vgpu, ch);
 		/*
 		 * SESSION 153-CONTINUED-75: heal a GR ring whose translation