vgsp: native publish-counter advance (sema_pub_adv; full-RAM content discovery of publish pages) - glxgears 120-148 FPS with no external pokes (SESSION 153 r936)

1 files changed, 211 insertions, 0 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 3d14d4b..c550843 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -6993,6 +6993,39 @@ MODULE_PARM_DESC(sema_gl_xlate,
 	"GL/3D (C597) direct stream -> synthesized GR candidate (default 0 = off)");
 
 /*
+ * SESSION 153-CONTINUED-4: native publish-counter advance (pace-wall fix).
+ *
+ * The GLX client's swap-completion ladder waits for the u32 at +0x8 of the
+ * driver-shared page whose struct magic is 0x0e836365 at +0x4 (the page is
+ * CPU-mapped from the X driver's memfd; r149-R3 watchpoint forensics named
+ * it "the publish counter our emulation never advances"). In a real stack
+ * the GPU completion of the client's kick channel (h=0xbfef00xx family)
+ * advances it to 3; ours never did, so every wait fell through the client's
+ * multi-second give-up ladder (140 FPS -> 0.045 FPS). A guest daemon write
+ * proved the value: u32 3 -> "702 frames in 5.0 seconds = 140.398 FPS".
+ *
+ * Native replacement: when the pump RETIRES a push of a kick-family channel
+ * (svc_replay_note consume path), write 3 to the page. The page moves in
+ * guest RAM per boot, so it is discovered once by content: the full 8 GiB
+ * RAM scan shows the magic page-aligned 16-byte signature
+ * (magic 0x0e836365 at +0x4, counter <= 3 at +0x8) matches exactly ONE page
+ * (verified unique on the r934 image before implementating). Discovery walks
+ * guest RAM descending (the driver's allocations live high), is rate-limited
+ * and attempt-capped, and every write is read back; a mismatch drops the
+ * discovered page so a stale/foreign target can never persist. Guarded by
+ * sema_pub_adv (default on; the family guard makes it inert for CUDA lanes)
+ * and fully logged.
+ */
+static bool gsp_sema_pub_adv = true;
+module_param_named(sema_pub_adv, gsp_sema_pub_adv, bool, 0644);
+MODULE_PARM_DESC(sema_pub_adv,
+	"Advance the GLX publish counter (u32 3) when a kick-channel push retires (default 1)");
+static u64 gsp_sema_pub_pa;
+module_param_named(sema_pub_pa, gsp_sema_pub_pa, ullong, 0644);
+MODULE_PARM_DESC(sema_pub_pa,
+	"Force the publish page PA for sema_pub_adv (0 = content-discover per boot)");
+
+/*
  * SESSION 149 R17 (M4): one recorded section upload command. The 3D-class
  * i2m shape is a 0x180 (LINE_LENGTH_IN) run naming the destination followed
  * by a 0x1b0 (LAUNCH_DMA + LOAD_INLINE_DATA) run carrying the payload.
@@ -20907,6 +20940,179 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu,
 	return cur >= ch->entry_want;
 }
 
+/* ------------------------------------------------------------------ */
+/* Native publish-counter advance (SESSION 153-CONTINUED-4, pace wall) */
+/* ------------------------------------------------------------------ */
+
+/*
+ * Two CPU-polled publish words gate the GLX client's per-frame pace; both
+ * live in the X driver's shared pages (CPU-mapped from its memfd) and both
+ * were proven to be needed: a guest daemon poking only the 0x44000-offset
+ * page sustained 140 FPS while the other word was already latched at 3;
+ * resetting both to 2 collapsed to the 22.4 s give-up ladder, and an
+ * ordered re-poke showed each word releasing part of the stall (page2 3 +
+ * main 2 -> still stalled; main 3 -> "603 frames in 5.0 seconds"). Each
+ * page is discovered by content and the counter word is at +0x8.
+ *
+ * Page signatures: a boot-varying magic at +0x4 (observed 0x0e836365 and
+ * 0xc228e806) is useless, but the handle words around it are stable:
+ * +0x10 family 0xc1d0xxxx and +0x14 == 0xcaf00006 on both observed boots.
+ * Page B's sparse notifier state is +0x208==0x80, +0x20c==1, +0x210==0.
+ * Both signatures were verified unique over the full 8 GiB RAM scan on the
+ * r935 image (and the legacy magic on r934) - exactly one match each.
+ */
+#define SVC_PUB_A_HANDLE_LO	0x14	/* stable client handle word */
+#define SVC_PUB_A_HANDLE_LO_V	0xcaf00006u
+#define SVC_PUB_A_HANDLE_HI	0x10	/* 0xc1d0xxxx family */
+#define SVC_PUB_A_HANDLE_MASK	0xfffff000u
+#define SVC_PUB_A_HANDLE_HI_V	0xc1d00000u
+#define SVC_PUB_A_MAGIC_OFF	0x4	/* legacy boot-varying magic */
+#define SVC_PUB_A_MAGIC		0x0e836365u
+#define SVC_PUB_B_OFF		0x208	/* sparse 2D notifier state */
+#define SVC_PUB_B_208		0x80u
+#define SVC_PUB_B_20C		0x1u
+#define SVC_PUB_WORD_OFF	0x8
+#define SVC_PUB_VALUE		3u
+#define SVC_PUB_SANE_MAX	0x100u	/* counter plausibility bound */
+#define SVC_PUB_SCAN_CAP	16	/* attempts before giving up per boot */
+
+static u64 svc_pub_pa_a;
+static u64 svc_pub_pa_b;
+static unsigned long svc_pub_next_scan;
+static u32 svc_pub_attempts;
+static u32 svc_pub_writes;
+
+/*
+ * Content discovery for both publish pages. The pages are created per
+ * session by the X driver and their GPAs move per boot, so find them by
+ * structure (the r934 full-RAM scan proved each signature matches exactly
+ * one page in all of guest RAM). Descending walk: the driver allocates in
+ * the high band, so the pages are reached within the first chunks.
+ */
+static void svc_pub_scan(struct nv_vgpu_device *vgpu)
+{
+	u8 *buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+	u64 gpa;
+
+	if (!buf)
+		return;
+	svc_pub_attempts++;
+	for (gpa = svc_pa_max() - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) {
+		if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) {
+			u32 o;
+
+			for (o = 0; o + SEMA_PAGE_SIZE <= SVC_SCAN_CHUNK;
+			     o += SEMA_PAGE_SIZE) {
+				u32 w4, w8, w10, w14, w208, w20c, w210;
+
+				if (!svc_pub_pa_a) {
+					memcpy(&w8, buf + o + SVC_PUB_WORD_OFF, 4);
+					if (w8 <= SVC_PUB_SANE_MAX) {
+						memcpy(&w4, buf + o + SVC_PUB_A_MAGIC_OFF, 4);
+						memcpy(&w10, buf + o + SVC_PUB_A_HANDLE_HI, 4);
+						memcpy(&w14, buf + o + SVC_PUB_A_HANDLE_LO, 4);
+						if (w4 == SVC_PUB_A_MAGIC ||
+						    (w14 == SVC_PUB_A_HANDLE_LO_V &&
+						     (w10 & SVC_PUB_A_HANDLE_MASK) ==
+						     SVC_PUB_A_HANDLE_HI_V))
+							svc_pub_pa_a = gpa + o;
+					}
+				}
+				if (!svc_pub_pa_b) {
+					memcpy(&w208, buf + o + SVC_PUB_B_OFF, 4);
+					if (w208 == SVC_PUB_B_208) {
+						memcpy(&w20c, buf + o + SVC_PUB_B_OFF + 4, 4);
+						memcpy(&w210, buf + o + SVC_PUB_B_OFF + 8, 4);
+						memcpy(&w8, buf + o + SVC_PUB_WORD_OFF, 4);
+						if (w20c == SVC_PUB_B_20C &&
+						    w210 == 0 &&
+						    w8 <= SVC_PUB_SANE_MAX)
+							svc_pub_pa_b = gpa + o;
+					}
+				}
+				if (svc_pub_pa_a && svc_pub_pa_b)
+					goto out;
+			}
+		}
+		if (gpa == 0)
+			break;
+	}
+out:
+	kfree(buf);
+	if (svc_pub_pa_a || svc_pub_pa_b)
+		pr_info("sema-pubadv: publish pages discovered a=0x%llx b=0x%llx (attempt %u)\n",
+			svc_pub_pa_a, svc_pub_pa_b, svc_pub_attempts);
+	else if (svc_pub_attempts <= 8)
+		pr_info("sema-pubadv: publish pages not found yet (attempt %u)\n",
+			svc_pub_attempts);
+}
+
+/*
+ * Write-3 + readback for one target. Returns false when the target is
+ * unreadable or non-coherent so a stale/foreign page can be dropped and
+ * re-discovered (never persist a wrong write target).
+ */
+static bool svc_pub_write_one(struct nv_vgpu_device *vgpu, u64 pa, u32 *back)
+{
+	u32 v = SVC_PUB_VALUE;
+	int rc;
+
+	*back = 0;
+	rc = svc_write(vgpu, pa + SVC_PUB_WORD_OFF, &v, sizeof(v));
+	if (!rc)
+		rc = svc_read(vgpu, pa + SVC_PUB_WORD_OFF, back, sizeof(*back));
+	return !rc && *back == v;
+}
+
+/*
+ * Called from svc_replay_note (pump retire path) for every consumed slot.
+ * Only kick-family channels (the GLX client's user channels, h=0xbfef00xx)
+ * drive the publish counters; every other lane is untouched.
+ */
+static void svc_pub_advance(struct svc_channel *ch)
+{
+	u32 back_a = 0, back_b = 0;
+	bool ok_a, ok_b;
+
+	if (!gsp_sema_pub_adv || !ch || !ch->vgpu)
+		return;
+	if (ch->h_object < 0xbfef0000u || ch->h_object > 0xbfef00ffu)
+		return;
+	if (!svc_pub_pa_a && !svc_pub_pa_b) {
+		if (gsp_sema_pub_pa) {
+			svc_pub_pa_a = gsp_sema_pub_pa &
+				       ~((u64)SEMA_PAGE_SIZE - 1);
+		} else if (svc_pub_attempts < SVC_PUB_SCAN_CAP &&
+			   time_after_eq(jiffies, svc_pub_next_scan)) {
+			svc_pub_scan(ch->vgpu);
+			svc_pub_next_scan = jiffies + 30 * HZ;
+		}
+		if (!svc_pub_pa_a && !svc_pub_pa_b)
+			return;
+	}
+	ok_a = !svc_pub_pa_a ||
+	       svc_pub_write_one(ch->vgpu, svc_pub_pa_a, &back_a);
+	ok_b = !svc_pub_pa_b ||
+	       svc_pub_write_one(ch->vgpu, svc_pub_pa_b, &back_b);
+	svc_pub_writes++;
+	if (!ok_a && !gsp_sema_pub_pa) {
+		pr_warn("sema-pubadv: page a write rejected (pa=0x%llx back=%u); dropping page\n",
+			svc_pub_pa_a, back_a);
+		svc_pub_pa_a = 0;
+		svc_pub_next_scan = jiffies;
+	}
+	if (!ok_b && !gsp_sema_pub_pa) {
+		pr_warn("sema-pubadv: page b write rejected (pa=0x%llx back=%u); dropping page\n",
+			svc_pub_pa_b, back_b);
+		svc_pub_pa_b = 0;
+		svc_pub_next_scan = jiffies;
+	}
+	if (svc_pub_writes <= 8 || (svc_pub_writes & 0x3ff) == 0)
+		pr_info("sema-pubadv: h=0x%x a=0x%llx/%s b=0x%llx/%s n=%u\n",
+			ch->h_object, svc_pub_pa_a, ok_a ? "3" : "ERR",
+			svc_pub_pa_b, ok_b ? "3" : "ERR", svc_pub_writes);
+}
+
 /* Mark a consumed GPFIFO slot for the current put epoch. */
 static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put)
 {
@@ -20939,6 +21145,11 @@ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put)
 	svc_opdef_finish_ch(ch, idx);
 	if (idx < SVC_GPFIFO_MAX_BITS)
 		set_bit(idx, ch->slot_done_epoch);
+	/*
+	 * SESSION 153-CONTINUED-4: retirement of a kick-family slot advances
+	 * the GLX client's publish counter (see svc_pub_advance).
+	 */
+	svc_pub_advance(ch);
 }
 
 /* Was this slot already consumed in the current put epoch? */