vgsp: canonical kick-push guard + anchor rescue chain + ring+0x20b0 notify targeting + per-handle notify cache - exec lane 48/48 continuous (SESSION 153 r977-r982)

1 files changed, 344 insertions, 6 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 15f728c..3bb883d 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -205,6 +205,19 @@ struct sema_map_entry {
 #define SVC_GR_RAM_REFRESH_MAX	3		/* keep-alive budget refreshes */
 #define SVC_GR_RAM_REFRESH_MS	10000		/* minimum spacing between them */
 #define SVC_GR_RAM_WIN		(2ULL << 20)	/* allocation window pinned */
+/*
+ * r979 (SESSION 153-CONTINUED-21): the kick family's per-frame submit
+ * record pushbuffer VA. Constant across every measured boot (r908
+ * documented the GL client ring pair {push=0x1203c0000,len=64} then
+ * {push=0x1203c0000,len=32}; r966-r978 every kick exec and every
+ * canonical push read named exactly this VA). Used to discriminate the
+ * kick ring page from shape lookalikes in the RAM ring scan - r978
+ * latched a foreign table page (pa=0x104fa9000) whose entry pair passed
+ * the shape gate and named a bogus 256 KB push (0x12048b824/len 0x40000)
+ * while the true ring (pa=0x1168ce000, found by scanning for the
+ * record's own entry pair) sat later in RAM scan order.
+ */
+#define SVC_GL_KICK_PUSH_VA	0x1203c0000ULL
 /* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */
 #define SVC_GR_USERD_WATCH	64
 /*
@@ -1202,6 +1215,10 @@ struct sema_svc {
 	u64			entry_norel_log;
 	u64			ops_denied;		/* work refused: implausible stream */
 	u64			false_trans;		/* resolvable VA mapped to non-push */
+	u64			kick_badpg;		/* r977: kick-family push reads
+						 * that lacked the canonical
+						 * submit-record prefix (wrong
+						 * page; forced the rescue) */
 	u64			rescue_map_ok;		/* SESSION 148: content-verified
 						 * map-rescue successes (GR + CE) */
 	u64			sem_write_reresolve;	/* writes re-resolved after read-back */
@@ -3502,6 +3519,7 @@ static void svc_pin_drop(u64 va)
  * every push was "implausible").
  */
 static inline bool svc_pool_sem_va(u64 va);
+static bool svc_kick_family(const struct svc_channel *ch);
 static void svc_pool_pa_forget(u64 va);
 
 /*
@@ -5575,9 +5593,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.gr_owed_resolved,
 		svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel,
 		svc.gr_stale_wedge, svc_pending_ovf);
-	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n",
+	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu kbpg=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n",
 		svc.counter_pub_calls, svc.gr_rel_applied,
 		svc.sem_write_reresolve, svc.pushpg_fallback,
+		svc.kick_badpg,
 		svc.pool_alt_applied, svc.pool_residue,
 		svc_wait_held, svc_wait_pass, svc_wait_giveup,
 		svc_wait_ovf);
@@ -7577,6 +7596,39 @@ MODULE_PARM_DESC(sema_kick_notify,
 	"Execute the GL client's per-frame kick NOTIFY (author the NvNotification at the SET_NOTIFY target VA; default 1, session 153 CONT.18)");
 static u32 svc_kick_log;	/* bounded kick-notify witness */
 static u64 svc_kick_count;	/* executor invocations */
+/*
+ * r982: the notify page a pending-marker hit proved, per knowledge the
+ * exec picked up. Kept in the service's own scope (not the channel
+ * snapshot): the kick channel is the only consumer and the entry is
+ * re-validated by a read before every use, so a stale PA can only cost
+ * one failed probe, never a write into freed memory.
+ */
+static struct {
+	u64 h;		/* channel handle that proved the page */
+	u64 pa;		/* PA of the marker-proven notify page */
+} svc_kick_pa_cache[4];
+
+static void svc_kick_pa_cache_store(u64 h, u64 pa)
+{
+	u32 i;
+
+	for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) {
+		if (svc_kick_pa_cache[i].h == h) {
+			svc_kick_pa_cache[i].pa = pa;
+			return;
+		}
+	}
+	for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) {
+		if (!svc_kick_pa_cache[i].h) {
+			svc_kick_pa_cache[i].h = h;
+			svc_kick_pa_cache[i].pa = pa;
+			return;
+		}
+	}
+	svc_kick_pa_cache[0].h = h;
+	svc_kick_pa_cache[0].pa = pa;
+}
+
 static bool gsp_sema_gl_draw_synth;
 module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644);
 MODULE_PARM_DESC(sema_gl_draw_synth,
@@ -7885,6 +7937,8 @@ static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len,
  * at +0xE. Resolution is the release path's trusted walk
  * (svc_lookup_trusted -> svc_write_ap), then readback verification.
  */
+static bool svc_kick_family(const struct svc_channel *ch);
+static u64 svc_push_pa_ceiling(void);
 static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu,
 				 struct svc_channel *ch, u64 va, u32 val,
 				 u64 push_va)
@@ -7895,6 +7949,8 @@ static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu,
 	bool ok = false, changed = false;
 	u32 o0 = 0, n0 = 0;
 	int rc = -ENOENT;
+	const char *rsrc = "trusted";
+	u32 kc = 0;
 
 	memset(nw, 0, sizeof(nw));
 	memcpy(nw, &ts, sizeof(ts));
@@ -7925,9 +7981,174 @@ static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu,
 			    svc_ap_of(ap2) == SVC_APERTURE_SYS) {
 				pa = pa2;
 				ap = ap2;
+				rsrc = "trusted-u";
 			}
 			svc_cur_vas = save;
 		}
+		/*
+		 * r978: r977's exec fires (27 calls) but authors the word in
+		 * the vidmem shadow (ap=2 pa=0x20e0b0) that no client reads,
+		 * while r973 - the boot where the kick consumption visibly
+		 * advanced the client - wrote the RING-ADJACENT sysmem slot
+		 * (exec pa=0x12ae340b0 over ring pa=0x12ae33000 = ring +
+		 * 0x1000 + 0xb0; the r975 probe measured the identical rule,
+		 * pa=0x104b640b0 over ring 0x104b63000). Both walk answers
+		 * miss that page this boot, so before falling back to the
+		 * shadow: (a) the general lookup (the discovered rm_mem map -
+		 * r979's sema_probe proved it resolves this VA to SYS after
+		 * the map rebuild, pa=0x1184a40b0), (b) a driver-declared pin
+		 * for this VA (the guest pin stream is the client's own
+		 * statement of the mapping), then (c) for a kick-family
+		 * channel the ring-relative completion slot, computed from
+		 * the live ring pin each call.
+		 */
+		if (svc_ap_of(ap) != SVC_APERTURE_SYS) {
+			u64 mpa = 0;
+			int map_ap = SVC_APERTURE_SYS;
+
+			if (!svc_lookup(vgpu, va, &mpa, &map_ap) &&
+			    svc_ap_of(map_ap) == SVC_APERTURE_SYS) {
+				pa = mpa;
+				ap = map_ap;
+				rsrc = "map";
+			}
+		}
+		if (svc_ap_of(ap) != SVC_APERTURE_SYS) {
+			u64 ipa = 0;
+			int iap = SVC_APERTURE_SYS;
+
+			if (!svc_ipin_lookup(va, &ipa, &iap) &&
+			    svc_ap_of(iap) == SVC_APERTURE_SYS) {
+				pa = ipa;
+				ap = iap;
+				rsrc = "ipin";
+			}
+		}
+		if (svc_ap_of(ap) != SVC_APERTURE_SYS && ch &&
+		    svc_kick_family(ch)) {
+			u64 ring_pa = 0;
+			int ring_ap = SVC_APERTURE_SYS;
+
+			if (!svc_pin_lookup_any(ch->gpfifo_va, &ring_pa,
+						&ring_ap) &&
+			    svc_ap_of(ring_ap) == SVC_APERTURE_SYS) {
+				u64 cand = (ring_pa &
+					    ~(u64)(SEMA_PAGE_SIZE - 1)) +
+					   SEMA_PAGE_SIZE +
+					   (va & (SEMA_PAGE_SIZE - 1));
+
+				if (cand < svc_push_pa_ceiling()) {
+					pa = cand;
+					ap = SVC_APERTURE_SYS;
+					rsrc = "ring+10b0";
+				}
+			}
+		}
+		/*
+		 * r981: the client's pending notification marker. r979/r980
+		 * measured the REAL slot's +0xC u32 reading 0x80000000
+		 * while the client waited (and 0 once completed; the write
+		 * below clears it) - on the page the discovered map names,
+		 * ring + 0x20b0 on both boots, one page ABOVE the r973-era
+		 * ring + 0x10b0 heuristic the exec had been writing
+		 * (0x1184a40b0 vs 0x1184a30b0 on r979; 0x12e07a0b0 vs
+		 * 0x12e0790b0 on r980 - the completion landed on the wrong
+		 * page both boots). When the chosen candidate does not show
+		 * the marker, probe the ring-relative alternatives and use
+		 * the one that does; the readback below still verifies the
+		 * write. A slot already completed (marker 0) keeps the
+		 * resolved candidate unchanged.
+		 *
+		 * r982: remember the page a pending-marker hit named, per
+		 * handle. r981's boot: exec n=1 found the marker page
+		 * (pa=0x129c1b0b0, ring+0x20b0) and completed it, but the
+		 * marker is 0 afterwards so every later exec fell back to
+		 * ring+0x10b0 (0x129c1a0b0 - wrong page, 33 further writes)
+		 * while the map/trusted stages were still catching up. The
+		 * cached page is retried at the top of the next call (read
+		 * first, used only while the read succeeds), so all later
+		 * completions stay on the page the marker proved.
+		 */
+		if (ch && svc_kick_family(ch)) {
+			/* Retry the page a marker hit proved, if still readable. */
+			for (kc = 0; kc < 4; kc++) {
+				u32 probe_buf = 0;
+
+				if (svc_kick_pa_cache[kc].h != ch->h_object ||
+				    !svc_kick_pa_cache[kc].pa)
+					continue;
+				if (!svc_read_ap(vgpu, svc_kick_pa_cache[kc].pa,
+						 SVC_APERTURE_SYS, &probe_buf,
+						 sizeof(probe_buf)) &&
+				    probe_buf != 0xffffffffu) {
+					pa = svc_kick_pa_cache[kc].pa;
+					ap = SVC_APERTURE_SYS;
+					rsrc = "kickcache";
+				}
+				break;
+			}
+		}
+		if (ch && svc_kick_family(ch)) {
+			u32 pend = 0;
+
+			if (!svc_read_ap(vgpu, pa + 0xc, svc_ap_of(ap),
+					 &pend, sizeof(pend)) &&
+			    pend != 0x80000000u) {
+				u64 ring_pa = 0;
+				int ring_ap = SVC_APERTURE_SYS;
+
+				if (!svc_pin_lookup_any(ch->gpfifo_va,
+							&ring_pa, &ring_ap) &&
+				    svc_ap_of(ring_ap) == SVC_APERTURE_SYS) {
+					u64 base = ring_pa &
+						~(u64)(SEMA_PAGE_SIZE - 1);
+					u32 off = (u32)(va & (SEMA_PAGE_SIZE - 1));
+					u64 cands[2];
+					int i;
+
+					cands[0] = base +
+						   2 * SEMA_PAGE_SIZE + off;
+					cands[1] = base +
+						   SEMA_PAGE_SIZE + off;
+					for (i = 0; i < 2; i++) {
+						u32 m = 0;
+
+						if (cands[i] >=
+						    svc_push_pa_ceiling())
+							continue;
+						if (!svc_read_ap(vgpu,
+								 cands[i] + 0xc,
+								 SVC_APERTURE_SYS,
+								 &m, sizeof(m)) &&
+						    m == 0x80000000u) {
+							pa = cands[i];
+							ap = SVC_APERTURE_SYS;
+							rsrc = i ?
+								"ring+10b0p" :
+								"ring+20b0p";
+							svc_kick_pa_cache_store(
+								ch->h_object,
+								cands[i]);
+							break;
+						}
+					}
+				}
+			} else if (pend == 0x80000000u &&
+				   svc_ap_of(ap) == SVC_APERTURE_SYS) {
+				/*
+				 * The chosen candidate IS the marker page:
+				 * prove it for the cache so later execs keep
+				 * writing the same page (r981: after the
+				 * single marker hit the fallback wrote two
+				 * wrong ring-relative pages for 33 further
+				 * execs). Only a SYS answer that showed the
+				 * marker is cached; a zero read (already
+				 * completed) caches nothing - unproven.
+				 */
+				svc_kick_pa_cache_store(ch->h_object, pa);
+			}
+		}
+		r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old));
 		r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old));
 		r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw));
 		if (!r1 && !r2 &&
@@ -7944,10 +8165,10 @@ static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu,
 		}
 	}
 	svc_kick_count++;
-	if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 511))) {
+	if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 127))) {
 		svc_kick_log++;
-		pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n",
-			ch ? ch->h_object : 0, push_va, va, pa, ap, val,
+		pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n",
+			ch ? ch->h_object : 0, push_va, va, pa, ap, rsrc, val,
 			(unsigned long long)ts, o0, n0, changed ? 1 : 0,
 			(unsigned long long)svc_kick_count, rc);
 	}
@@ -19420,6 +19641,22 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 				continue;
 			if (len1 < 4 || len1 > 0x100000)
 				continue;
+			/*
+			 * r979: kick-family ring discrimination. Shape alone
+			 * admitted a foreign table page ahead of the real ring
+			 * on r978 (pa=0x104fa9000, entries naming a bogus
+			 * 0x12048b824/0x40000 push) while the true ring's
+			 * entry pair names the client's per-frame record VA
+			 * (SVC_GL_KICK_PUSH_VA, the r908-documented GL
+			 * pushbuffer, constant across r908-r978). A
+			 * kick-family candidate must name the record; the
+			 * lookalike class then never latches. Non-kick
+			 * channels keep the shape gate unchanged.
+			 */
+			if (svc_kick_family(ch) &&
+			    va0 != SVC_GL_KICK_PUSH_VA &&
+			    va1 != SVC_GL_KICK_PUSH_VA)
+				continue;
 			*pa_out = gpa + o;
 			return true;
 		}
@@ -20114,6 +20351,27 @@ static const u8 svc_gr_anchor_sig[16] = {
 };
 #define SVC_GR_ANCHOR_SWEEPS	3
 
+/*
+ * r977 (SESSION 153-CONTINUED-21): the kick family's per-frame submit
+ * record is a fixed method stream opening with the SET_SUBDEVICE_MASK
+ * pair 0x0001fff0 0x20010000 (LE bytes f0 ff 01 00 00 00 01 20; the
+ * bytes r975's anchor pin answered and r973 decoded to n=512 execs).
+ * Every boot r966-r976 read that VA through some untrusted translation
+ * and, because a plausible-looking garbage page (r976: first8=9864be60
+ * ffffffff at pa=0x10353b000 via the src=3 window guess) passes the
+ * generic method-stream parse, the entry was consumed with the wrong
+ * bytes and the kick lane stayed silent for the whole session. For
+ * this family the canonical prefix IS the content identity: a read
+ * that lacks it came from a stale/foreign page no matter how
+ * plausible it parses, and must take the rescue chain (VAS walk ->
+ * content anchor -> pin).
+ */
+static bool svc_kick_canonical(const u8 *buf, size_t len)
+{
+	return len >= sizeof(svc_gr_anchor_pre) &&
+	       !memcmp(buf, svc_gr_anchor_pre, sizeof(svc_gr_anchor_pre));
+}
+
 static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu,
 			       struct svc_channel *ch, u64 va, u32 len)
 {
@@ -31610,6 +31868,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					}
 					bool injected = false;
 					bool allow_ops = (ch->engine_type != 0);
+					bool kick_badpg = false;
 
 					/*
 					 * A resolved buffer that does not start
@@ -31709,7 +31968,48 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							len >= 4 && len <= 16 &&
 							!memchr_inv(pb, 0, len);
 
-						if (!svc_method_stream_plausible(pb, len) &&
+						/*
+						 * r977/r979: a read of the
+						 * kick family's per-frame
+						 * record (SVC_GL_KICK_PUSH_VA;
+						 * r978 showed the channel also
+						 * carries bulk pushes that
+						 * must NOT be held to this
+						 * content identity - a 256 KB
+						 * push at 0x12048b824) whose
+						 * bytes lack the canonical
+						 * submit-record prefix is a
+						 * wrong page no matter how
+						 * plausible it parses (r976:
+						 * src=3 window garbage
+						 * 9864be60 ffffffff consumed
+						 * as a valid push and the kick
+						 * lane ran zero execs). Force
+						 * the implausible path so the
+						 * VAS walk / content anchor /
+						 * pin-drop chain below
+						 * re-anchors the page.
+						 */
+						kick_badpg = !zero_nop &&
+							svc_kick_family(ch) &&
+							va == SVC_GL_KICK_PUSH_VA &&
+							len <= 128 &&
+							!svc_kick_canonical(pb, len);
+						if (kick_badpg) {
+							svc.kick_badpg++;
+							if (svc_pushpg_fb_log < 64) {
+								svc_pushpg_fb_log++;
+								pr_info("sema-kickpg: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx first8=%08x %08x (non-canonical kick page; rescue chain)\n",
+									ch->h_object, va, len,
+									(unsigned)svc_read_last_src,
+									svc_read_last_pa,
+									len >= 4 ? *(const u32 *)pb : 0,
+									len >= 8 ? *(const u32 *)(pb + 4) : 0);
+							}
+						}
+
+						if ((!svc_method_stream_plausible(pb, len) ||
+						     kick_badpg) &&
 						    !zero_nop) {
 							svc.false_trans++;
 							if (injected) {
@@ -31787,6 +32087,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									bool pl = !rr ? svc_method_stream_plausible(vb, len)
 										      : false;
 
+									/*
+									 * r977: for a kick-family
+									 * entry a plausible walk
+									 * answer still has to carry
+									 * the canonical prefix - a
+									 * wrong-VAS garbage page
+									 * parses plausibly (r976) and
+									 * would re-shadow the real
+									 * page here.
+									 */
+									if (pl && kick_badpg &&
+									    !svc_kick_canonical(vb, len))
+										pl = false;
+
 									if (gsp_sema_dbg &&
 									    svc_pushpg_fb_log < 64) {
 										svc_pushpg_fb_log++;
@@ -31800,6 +32114,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 										memcpy(pb, vb, len);
 										svc.pushpg_fallback++;
 										ok = true;
+										/*
+										 * r977: a kick-family
+										 * entry whose page the
+										 * walk re-anchored is
+										 * pinned so later passes
+										 * read the canonical page
+										 * directly (src=2) instead
+										 * of re-paying the walk
+										 * while the content anchor
+										 * catches up (r975 needed
+										 * ~78 s for the anchor; the
+										 * walk answered in seconds).
+										 */
+										if (kick_badpg &&
+										    wap == SVC_APERTURE_SYS &&
+										    svc_kick_canonical(vb, len))
+											svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1),
+												      wpa,
+												      SVC_APERTURE_SYS);
 										if (gsp_sema_dbg &&
 										    svc_pushpg_fb_log < 64) {
 											svc_pushpg_fb_log++;
@@ -31890,7 +32223,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								      va, pb,
 								      len) &&
 						    svc_method_stream_plausible(pb,
-										len)) {
+										len) &&
+						    (!(svc_kick_family(ch) &&
+						       va == SVC_GL_KICK_PUSH_VA &&
+						       len <= 128) ||
+						     svc_kick_canonical(pb,
+									len))) {
 							ch->rescue_last = 0;
 							ch->push_hold_until = 0;
 							svc.gpfifo_rescued++;