vgsp-sema: SESSION 129 util-channel LOAD_INLINE_DATA arbitration + witness - the subchannel-2 arm never fired (r737 writes=0) because the utility channel's read path could not produce a parseable methods entry (the tail-served split shape); the util sweep now arbitrates attempt A (channel path, unchanged) against attempt B (direct registered-VAS walk, the walk_probe-proven path) and lets the strict parse choose the stream, a still-zero tail-served payload is refused (no once-store, retried every pass; in-push payloads untouched), and a bounded deduplicated sema-utildec witness names per distinct entry: winning attempt, both parses, tail VA/length, LID header count/subch and the arm's saw/fired/zero state

1 files changed, 258 insertions, 37 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index f77b75d..c1426d4 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -24082,6 +24082,35 @@ static u32 svc_gr_ahead_req_log;
 static u64 svc_gr_ahead_inline_writes;
 
 /*
+ * SESSION 129 util-channel witness: the utility channel's subchannel-2
+ * LOAD_INLINE_DATA arm never fired (writes=0 on r737) while the same arm
+ * fired for in-push payloads on the GR channel - i.e. the failure sits in
+ * the split (tail-served) shape. This struct carries the per-entry story
+ * out of the translator for one bounded, deduplicated witness line:
+ * which read source answered (attempt A = the channel path, 1 = the direct
+ * registered-VAS walk), whether the strict parse accepted the entry, and
+ * what the LID arm did (saw/fired/refused a still-zero payload).
+ */
+struct svc_util_wit {
+	u32 p_src;	/* attempt that won: 0 = channel, 1 = VAS walk, 0xff none */
+	u32 pa_ok;	/* attempt-A parse (method count) */
+	u32 pb_ok;	/* attempt-B parse (method count) */
+	u32 t_src;	/* tail read source: 0 = channel, 1 = walk, 0xff none */
+	u32 t_len;	/* tail length served */
+	bool saw_lid;	/* a LOAD_INLINE_DATA arm was reached */
+	bool fired;	/* ... and authored its write */
+	bool zero;	/* ... but the tail payload was still all-zero (pending) */
+	u32 dst;
+	u32 line_len;
+	u32 line_lc;
+	u32 count;
+};
+static struct svc_util_wit svc_util_wit;
+static u64 svc_util_pending_zero;
+static u32 svc_util_dec_log;
+static struct { u64 va; u64 h; } svc_util_dec_keys[32];
+
+/*
  * SESSION 127 C5C0 translator, bounded tail materialization buffer: a
  * LOAD_INLINE_DATA run whose payload overruns the entry end is served from
  * the next ring entry (the driver splits large inline runs into a methods
@@ -24380,6 +24409,8 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 			u32 cap = count * 4;
 			u32 bytes, k, words;
 			u64 h = svc_push_hash(buf, len);
+			bool tail_served = end > len;
+			bool refuse_zero = false;
 
 			if (nbytes > cap)
 				nbytes = cap;
@@ -24399,11 +24430,39 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 							     taillen,
 							     off + 4 + 4 * k) },
 				       4);
-			if (bytes && !svc_inline_once_seen(push_va, h, off_out,
-							   bytes)) {
+			/*
+			 * SESSION 129: a tail-served payload whose head is still
+			 * all zeros means the tail source answered with an empty
+			 * page (a stale pin shadow, or the driver has not staged
+			 * the bytes yet) - authoring zeros over the destination
+			 * would corrupt the code heap. Leave the write pending
+			 * (no once-store, the sweep retries every pass) until a
+			 * nonzero payload arrives. In-push payloads are never
+			 * gated: the bytes are part of the flushed submit.
+			 */
+			if (inline_only && tail_served && bytes &&
+			    !memchr_inv(svc_ahead_inline_buf, 0,
+					min_t(u32, bytes, 16)))
+				refuse_zero = true;
+			if (inline_only) {
+				svc_util_wit.saw_lid = true;
+				svc_util_wit.dst = (u32)off_out;
+				svc_util_wit.line_len = line_len;
+				svc_util_wit.line_lc = line_count;
+				svc_util_wit.count = count;
+				if (refuse_zero)
+					svc_util_wit.zero = true;
+			}
+			if (refuse_zero) {
+				svc_util_pending_zero++;
+			} else if (bytes && !svc_inline_once_seen(push_va, h,
+								  off_out,
+								  bytes)) {
 				svc_inline_once_store(push_va, h, off_out,
 						      bytes);
 				svc_gr_ahead_inline_writes++;
+				if (inline_only)
+					svc_util_wit.fired = true;
 				(void)svc_grce_inline_write(vgpu, off_out,
 							    svc_ahead_inline_buf,
 							    bytes,
@@ -24422,12 +24481,19 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 					&svc_grce_dst_ring[svc_grce_dst_ring_next %
 							  SVC_GRCE_DST_MAX];
 
-				rec->dst_va = off_out;
-				rec->len = min_t(u32, bytes,
-						 GSP_BRIDGE_INLINE_BYTES);
-				memcpy(rec->data, svc_ahead_inline_buf,
-				       rec->len);
-				svc_grce_dst_ring_next++;
+				/*
+				 * A refused (still-zero) tail payload is not
+				 * recorded either: the next pass re-reads the
+				 * tail and the real bytes land then.
+				 */
+				if (!refuse_zero) {
+					rec->dst_va = off_out;
+					rec->len = min_t(u32, bytes,
+							 GSP_BRIDGE_INLINE_BYTES);
+					memcpy(rec->data, svc_ahead_inline_buf,
+					       rec->len);
+					svc_grce_dst_ring_next++;
+				}
 			}
 			if (end > len)
 				break;
@@ -24692,6 +24758,63 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu,
 }
 
 /*
+ * SESSION 129: read a VA straight through the registered-VAS walks only
+ * (newest registration first), one page at a time - the exact path
+ * walk_probe and sema_probe prove. The util sweep's channel read can be
+ * shadowed by a stale injected pin / window guess for the utility VAs
+ * (observed r737: the subchannel-2 LID arm never fired although the push
+ * page content is real and the VAS walk resolves both the methods entry
+ * and its payload slot); this helper is the arbitration fallback that
+ * bypasses those sources entirely.
+ */
+static int svc_util_walk_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+			      size_t len)
+{
+	u8 *p = buf;
+
+	while (len) {
+		u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+		size_t off = va & (SEMA_PAGE_SIZE - 1);
+		size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - off);
+		u64 pa = 0;
+		int ap = SVC_APERTURE_SYS;
+
+		if (svc_walk_vaspaces(vgpu, page, &pa, &ap))
+			return -ENOENT;
+		if (svc_read_ap(vgpu, pa + off, svc_ap_of(ap), p, chunk))
+			return -EIO;
+		p += chunk;
+		va += chunk;
+		len -= chunk;
+	}
+	return 0;
+}
+
+/*
+ * SESSION 129: does this entry window carry a LOAD_INLINE_DATA header? Used
+ * by the bounded util-sweep witness to select only the entries that can
+ * exercise the arm (one per distinct content, so a stable ring logs once).
+ */
+static u32 svc_util_lid_scan(const u8 *buf, u32 len, u32 *first_subch)
+{
+	u32 k, n = 0;
+
+	*first_subch = 0xff;
+	for (k = 0; k + 4 <= len; k += 4) {
+		u32 hdr;
+
+		memcpy(&hdr, buf + k, 4);
+		if (((hdr >> 29) & 0x7) == 3 &&
+		    ((hdr & 0x1fff) << 2) == GRCE_M_LOAD_INLINE_DATA) {
+			if (*first_subch == 0xff)
+				*first_subch = (hdr >> 13) & 0x7;
+			n++;
+		}
+	}
+	return n;
+}
+
+/*
  * SESSION 128: utility-channel inline-download materialization.
  *
  * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D
@@ -24713,6 +24836,14 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu,
  * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring
  * costs only the reads. Requires sema_gr_xlate; bounded by
  * sema_gr_util_scan entries per channel per pass (0 = off).
+ *
+ * SESSION 129 read arbitration: attempt A is the channel path (unchanged);
+ * when A fails to read, or reads content the strict parse rejects, attempt
+ * B re-reads the push and the tail through svc_util_walk_read and the
+ * parse arbitrates - a source that yields a method stream wins. The r737
+ * evidence (arm never fired while the content is real) pointed at exactly
+ * this: a shadowed channel read consumed the entry without a parseable
+ * buffer while the walk resolves the true pages.
  */
 static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
 			      struct svc_channel *ch)
@@ -24726,6 +24857,7 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
 #define SVC_UTIL_BUF (64 * 1024)
 	static u8 ubuf[SVC_UTIL_BUF];
 	static u8 nbuf[SVC_UTIL_BUF];
+	static u8 bbuf[SVC_UTIL_BUF];
 	static u64 svc_gr_util_entries;
 	static u64 svc_gr_util_writes;
 	static u32 svc_gr_util_log;
@@ -24737,7 +24869,7 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
 		return;
 	for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) {
 		u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE];
-		u32 e0, e1, len, rlen, tail_len = 0;
+		u32 e0, e1, len, rlen;
 		u64 va, before;
 
 		if (svc_read_va(vgpu,
@@ -24756,40 +24888,129 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
 		if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI)
 			continue;
 		rlen = min_t(u32, len, sizeof(ubuf));
-		if (svc_read_va_push(vgpu, ch, va, ubuf, rlen))
-			continue;
 		/*
-		 * The payload of a trailing LOAD_INLINE_DATA run rides the
-		 * next GPFIFO entry (the driver's source-view GPFIFO call,
-		 * observed beef0100 idx00->idx01 and idx08->idx09): read it
-		 * as the tail exactly like svc_gr_ahead_sweep does.
+		 * SESSION 129 read arbitration + bounded witness (see the
+		 * sweep's header comment). Attempt A: the channel path,
+		 * unchanged. Attempt B: the direct registered-VAS walk, used
+		 * when A cannot read or cannot parse; the strict parse then
+		 * arbitrates which content is the method stream.
 		 */
-		if (svc_read_va(vgpu,
-				ch->gpfifo_va +
-				(u64)((i + 1) % ch->gpfifo_entries) *
-				      SVC_GPFIFO_ENTRY_SIZE,
-				nent, sizeof(nent)) == 0) {
-			u32 ne0, ne1, nlen;
-			u64 nva;
+		{
+			u32 push_rc, parse_a = 0, parse_b = 0;
+			u32 tl_a = 0, tl_b = 0;
+			u32 lidn, lidsc = 0xff, slot;
+			u64 nva = 0;
+			u32 nlen = 0;
+			u64 hkey;
+			u8 *pbuf = ubuf;
+			bool ut = false, seen = false;
+
+			push_rc = svc_read_va_push(vgpu, ch, va, ubuf,
+						   rlen);
+			/*
+			 * The payload of a trailing LOAD_INLINE_DATA run rides
+			 * the next GPFIFO entry (the driver's source-view
+			 * GPFIFO call, observed beef0100 idx00->idx01 and
+			 * idx08->idx09): read it as the tail exactly like
+			 * svc_gr_ahead_sweep does.
+			 */
+			if (svc_read_va(vgpu,
+					ch->gpfifo_va +
+					(u64)((i + 1) % ch->gpfifo_entries) *
+					      SVC_GPFIFO_ENTRY_SIZE,
+					nent, sizeof(nent)) == 0) {
+				u32 ne0, ne1;
 
-			memcpy(&ne0, nent, 4);
-			memcpy(&ne1, nent + 4, 4);
-			nva = ((u64)(ne1 & 0xff) << 32) |
-			      (u64)(ne0 & 0xfffffffc);
-			nlen = ((ne1 >> 10) & 0x1fffff) << 2;
-			if (nva && nlen && nlen < SVC_PUSH_MAX) {
-				tail_len = min_t(u32, nlen, sizeof(nbuf));
+				memcpy(&ne0, nent, 4);
+				memcpy(&ne1, nent + 4, 4);
+				nva = ((u64)(ne1 & 0xff) << 32) |
+				      (u64)(ne0 & 0xfffffffc);
+				nlen = ((ne1 >> 10) & 0x1fffff) << 2;
+			}
+			if (!push_rc && nva && nlen && nlen < SVC_PUSH_MAX) {
+				tl_a = min_t(u32, nlen, sizeof(nbuf));
 				if (svc_read_va_push(vgpu, ch, nva, nbuf,
-						     tail_len))
-					tail_len = 0;
+						     tl_a))
+					tl_a = 0;
+			}
+			if (!push_rc)
+				parse_a = svc_ahead_parse(ubuf, rlen, nbuf,
+							  tl_a, &ut);
+			if (push_rc || !parse_a) {
+				bool bpush = false;
+
+				if (nva && nlen && nlen < SVC_PUSH_MAX) {
+					u32 want = min_t(u32, nlen,
+							 sizeof(nbuf));
+
+					if (!svc_util_walk_read(vgpu, nva,
+								nbuf,
+								want))
+						tl_b = want;
+				}
+				if (!svc_util_walk_read(vgpu, va, bbuf,
+							rlen)) {
+					bpush = true;
+					parse_b = svc_ahead_parse(bbuf, rlen,
+								  nbuf, tl_b,
+								  &ut);
+					/*
+					 * A's buffer is authoritative only
+					 * when A actually read it; otherwise
+					 * (or when only B parses) B's walk
+					 * content is the stream.
+					 */
+					if (push_rc ||
+					    (parse_b && !parse_a)) {
+						pbuf = bbuf;
+						tl_a = tl_b;
+					}
+				}
+				if (push_rc && !bpush)
+					continue;	/* unreadable both ways */
 			}
+			memset(&svc_util_wit, 0, sizeof(svc_util_wit));
+			before = svc_gr_ahead_inline_writes;
+			(void)svc_gr_ahead_push(vgpu, ch, pbuf, rlen, nbuf,
+						tl_a, va, true);
+			if (svc_gr_ahead_inline_writes != before)
+				nmade++;
+			swept++;
+			/*
+			 * Bounded witness, one line per distinct LID-carrying
+			 * (or unparseable) entry content: which attempt was
+			 * used, the parses, the tail resolution and what the
+			 * LID arm did.
+			 */
+			lidn = svc_util_lid_scan(pbuf, rlen, &lidsc);
+			if (!lidn && parse_a)
+				continue;
+			hkey = svc_push_hash(pbuf, rlen);
+			for (slot = 0; slot < 32; slot++)
+				if (svc_util_dec_keys[slot].h == hkey &&
+				    svc_util_dec_keys[slot].va == va) {
+					seen = true;
+					break;
+				}
+			if (seen || svc_util_dec_log >= 96)
+				continue;
+			slot = svc_util_dec_log & 31;
+			svc_util_dec_keys[slot].va = va;
+			svc_util_dec_keys[slot].h = hkey;
+			svc_util_dec_log++;
+			pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n",
+				ch->h_object, i, va, rlen,
+				pbuf == bbuf ? 1u : 0u, parse_a, parse_b,
+				nva, tl_a, lidn, lidsc,
+				svc_util_wit.saw_lid ? 1 : 0,
+				svc_util_wit.fired ? 1 : 0,
+				svc_util_wit.zero ? 1 : 0,
+				svc_util_wit.dst, svc_util_wit.line_len,
+				svc_util_wit.line_lc,
+				rlen >= 4 ? *(const u32 *)pbuf : 0,
+				rlen >= 8 ? *(const u32 *)(pbuf + 4) : 0);
 		}
-		before = svc_gr_ahead_inline_writes;
-		(void)svc_gr_ahead_push(vgpu, ch, ubuf, rlen, nbuf,
-					tail_len, va, true);
-		if (svc_gr_ahead_inline_writes != before)
-			nmade++;
-		swept++;	}
+	}
 	svc_gr_util_entries += swept;
 	svc_gr_util_writes += nmade;
 	if (swept && (nmade || svc_gr_util_log < 24)) {