vgsp-sema: SESSION 130 util-channel LOAD_INLINE_DATA arm FIXED - the subch-2 GRCE operand group was swallowed by the TURING-copy branch (its `subch >= 2` guard also matches subchannel 2, leaving have_out/line_len unset so the LID arm never fired: r739 witness `lidn=1 subch=2 saw=0 dst=0 ll=0`); the 0x180..0x18c group now parses FIRST for subch 1|2, plus tail-content arbitration (a zero-headed channel tail is re-read through the direct registered-VAS walk before the arm's zero-refusal can stall the upload) and the witness now carries the served tail's first two words + the alt-read counter

1 files changed, 77 insertions, 39 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index c1426d4..4984a8c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -24097,6 +24097,7 @@ struct svc_util_wit {
 	u32 pb_ok;	/* attempt-B parse (method count) */
 	u32 t_src;	/* tail read source: 0 = channel, 1 = walk, 0xff none */
 	u32 t_len;	/* tail length served */
+	u32 t0, t1;	/* SESSION 130: first two words of the served tail */
 	bool saw_lid;	/* a LOAD_INLINE_DATA arm was reached */
 	bool fired;	/* ... and authored its write */
 	bool zero;	/* ... but the tail payload was still all-zero (pending) */
@@ -24501,8 +24502,49 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 			continue;
 		}
 
-		/* TURING_DMA_COPY_A operand runs (subch >= 2). */
-		if (xlate && subch >= 2) {
+		/*
+		 * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c), for
+		 * subchannel 1 AND 2. The mem2mem subchannel (2, the
+		 * A140/KEPLER_INLINE_TO_MEMORY_B upload lane the utility
+		 * channels drive) carries the same operand methods as the
+		 * compute subchannel (1). This group MUST be parsed BEFORE
+		 * the TURING-copy branch below: that branch's `subch >= 2`
+		 * guard also matches subchannel 2 and would swallow the
+		 * 0x180..0x18c methods unread, leaving have_out/line_len
+		 * unset - the exact reason the utility channel's subch-2
+		 * LOAD_INLINE_DATA arm never fired (r737/r739 witness:
+		 * `lidn=1 subch=2 ... saw=0 dst=0 ll=0`).
+		 */
+		if (xlate && (subch == 1 || subch == 2) &&
+		    addr >= 0x180 && addr <= 0x18c && count) {
+			u32 k, n = count > 4 ? 4 : count;
+			u32 vv[4] = { w0, w1, w2, w3 };
+
+			for (k = 0; k < n; k++) {
+				switch (addr + 4 * k) {
+				case 0x180:
+					line_len = vv[k];
+					break;
+				case 0x184:
+					line_count = vv[k];
+					break;
+				case 0x188:
+					off_out = (off_out & 0xffffffffULL) |
+						  ((u64)(vv[k] & 0x1ffff)
+						   << 32);
+					have_out = 1;
+					break;
+				case 0x18c:
+					off_out = (off_out &
+						   ~0xffffffffULL) | vv[k];
+					have_out = 1;
+					break;
+				default:
+					break;
+				}
+			}
+		} else if (xlate && subch >= 2) {
+			/* TURING_DMA_COPY_A operand runs (subch >= 2). */
 			u32 k, n = count > 4 ? 4 : count;
 			u32 vv[4] = { w0, w1, w2, w3 };
 
@@ -24598,41 +24640,6 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 					}
 				}
 			}
-		} else if (xlate && (subch == 1 || subch == 2) &&
-			   addr >= 0x180 && addr <= 0x18c && count) {
-			u32 k, n = count > 4 ? 4 : count;
-			u32 vv[4] = { w0, w1, w2, w3 };
-
-			/*
-			 * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c).
-			 * The mem2mem subchannel (2, the A140/KEPLER_INLINE_
-			 * TO_MEMORY_B upload lane the utility channels drive)
-			 * carries the same operand methods as the compute
-			 * subchannel (1).
-			 */
-			for (k = 0; k < n; k++) {
-				switch (addr + 4 * k) {
-				case 0x180:
-					line_len = vv[k];
-					break;
-				case 0x184:
-					line_count = vv[k];
-					break;
-				case 0x188:
-					off_out = (off_out & 0xffffffffULL) |
-						  ((u64)(vv[k] & 0x1ffff)
-						   << 32);
-					have_out = 1;
-					break;
-				case 0x18c:
-					off_out = (off_out &
-						   ~0xffffffffULL) | vv[k];
-					have_out = 1;
-					break;
-				default:
-					break;
-				}
-			}
 		}
 
 		if (!inline_only && addr == HOST_M_SEM_ADDR_LO && count >= 5) {
@@ -24858,9 +24865,13 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
 	static u8 ubuf[SVC_UTIL_BUF];
 	static u8 nbuf[SVC_UTIL_BUF];
 	static u8 bbuf[SVC_UTIL_BUF];
+	static u8 tbuf[SVC_UTIL_BUF];
 	static u64 svc_gr_util_entries;
 	static u64 svc_gr_util_writes;
 	static u32 svc_gr_util_log;
+	static u64 svc_util_tail_alt;	/* SESSION 130: tails re-read through
+					 * the direct walk because the
+					 * channel read was zero-headed */
 	u32 i, maxent = min_t(u32, gsp_sema_gr_util_scan, 64);
 	u32 swept = 0, nmade = 0;
 
@@ -24969,7 +24980,33 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
 				if (push_rc && !bpush)
 					continue;	/* unreadable both ways */
 			}
+			/*
+			 * SESSION 130 tail-content arbitration: a tail-served
+			 * payload whose channel head is still all-zero is not
+			 * the staged bytes (the page the channel translation
+			 * points at is empty, or the read is shadowed by a
+			 * stale pin). Try the direct registered-VAS walk -
+			 * the same path attempt B proves - before handing
+			 * the tail to the LID arm, whose zero-headed refusal
+			 * would otherwise stall the upload every pass.
+			 */
+			if (tl_a && nva &&
+			    !memchr_inv(nbuf, 0, min_t(u32, tl_a, 16))) {
+				u32 want = tl_a;
+
+				if (!svc_util_walk_read(vgpu, nva, tbuf,
+							want) &&
+				    memchr_inv(tbuf, 0,
+					       min_t(u32, want, 16))) {
+					memcpy(nbuf, tbuf, want);
+					svc_util_tail_alt++;
+				}
+			}
 			memset(&svc_util_wit, 0, sizeof(svc_util_wit));
+			if (tl_a >= 4)
+				memcpy(&svc_util_wit.t0, nbuf, 4);
+			if (tl_a >= 8)
+				memcpy(&svc_util_wit.t1, nbuf + 4, 4);
 			before = svc_gr_ahead_inline_writes;
 			(void)svc_gr_ahead_push(vgpu, ch, pbuf, rlen, nbuf,
 						tl_a, va, true);
@@ -24998,10 +25035,11 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
 			svc_util_dec_keys[slot].va = va;
 			svc_util_dec_keys[slot].h = hkey;
 			svc_util_dec_log++;
-			pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n",
+			pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x t8=%08x %08x alt=%llu lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n",
 				ch->h_object, i, va, rlen,
 				pbuf == bbuf ? 1u : 0u, parse_a, parse_b,
-				nva, tl_a, lidn, lidsc,
+				nva, tl_a, svc_util_wit.t0, svc_util_wit.t1,
+				svc_util_tail_alt, lidn, lidsc,
 				svc_util_wit.saw_lid ? 1 : 0,
 				svc_util_wit.fired ? 1 : 0,
 				svc_util_wit.zero ? 1 : 0,