sema-svc: validate pushes with the full method encoding; deliver releases the sweep already applied

The runtime client's second push on each UVM copy channel starts with a
NON_INC method (0x60080002, count 8) and its release method sits 0x50 bytes
in. The stream validator accepted only INC (op 1) headers, so the readable
page failed the plausibility gate and every content-rescue candidate for
that VA failed the read-back validation - the client's tracking semaphore
for the push never advanced and the runtime init parked.

- one shared parser (svc_stream_parse) accepts INC/NON_INC (op 1/3),
  IMMD_DATA (op 4, header-only), ONE_INC (op 5) and END_PB_SEGMENT (op 7),
  and still requires an exact method boundary over the whole entry plus
  trailing zero padding - the strict end condition is what keeps the
  content rescue safe against firmware/allocator pages
- svc_method_stream_plausible and svc_stream_full_ok both use it, so a real
  readable push is decoded directly and a rescued page validates
- the decoder skips NON_INC/ONE_INC methods whole (including their data
  words) instead of resyncing word-by-word through the payload, which could
  misread data as headers and queue bogus work

Follow-on: the content sweep applies releases from RAM pages ahead of the
pump, so the pump's expected payload was already satisfied when it examined
the entry; the CE grace path then held each entry for the full 8 s window
before consuming it - one grace period per push, serializing the client's
runtime init. The pump now records the expected payload when it first
examines an entry (entry_want) and consumes the entry immediately once the
pool advances past it, on both the unreadable-rescue and the
readable-no-release paths.

Evidence (fresh boot, real releases, watchers + kprobes, no fakes):
cudaGetDeviceCount rc=0 in the runtime client, device-attribute walk
proceeds, pool semaphores advance; before this change the client sat in the
0x20801702 service-interrupt loop for >6 min and never left device init.

1 files changed, 146 insertions, 35 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 1ea8038..58778c2 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -149,6 +149,14 @@ struct svc_channel {
 				 * entries, and an INC release must not advance
 				 * the tracking semaphore twice for one push */
 	u32	expected_sem_payload;	/* next semaphore payload we expect for this channel's tracking sem */
+	u32	entry_want;	/* expected_sem_payload when the pending GPFIFO
+				 * entry was first examined: if the pool has
+				 * advanced past it since, the entry's release
+				 * was already delivered by another path (the
+				 * content sweep) and the entry can be consumed
+				 * immediately instead of waiting out the grace
+				 * window (observed: one 8 s grace per push,
+				 * serializing the runtime init) */
 	u64	last_rel_va;	/* last tracking-sem VA seen in a readable push */
 	u32	logged_expected;	/* last pool value logged (rate limit) */
 	unsigned long rescue_last;	/* rate limit for the full-RAM rescue scan */
@@ -2429,23 +2437,72 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 }
 
 /*
- * Does this buffer start like a push? The discovered VA->PA map is
+ * Parse a buffer as a complete push method stream.
+ *
+ * The B06F/C5B5 method encoding carries four method shapes: SEC_OP 1
+ * (INC_METHOD), 3 (NON_INC_METHOD), 4 (IMMD_DATA_METHOD, the immediate data
+ * lives in the header's count field so it has no payload words), 5
+ * (ONE_INC) and 7 (END_PB_SEGMENT, header only); a zero word is padding.
+ * UVM emits INC for almost everything but a CE push can *start* with a
+ * NON_INC method (observed: a runtime channel's second push begins
+ * `0x60080002` = NON_INC count 8), so a parser that only accepts op 1
+ * rejects real, readable pushes and no content rescue can validate their
+ * page.
+ *
+ * Returns true when every word is covered by a method (trailing zero
+ * padding allowed) and at least one method was seen. Arbitrary data
+ * (firmware, page tables, allocator lists) practically never parses to an
+ * exact method boundary, which is what keeps the content rescue safe.
+ */
+static bool svc_stream_parse(const u8 *buf, u32 len)
+{
+	u32 off = 0, methods = 0;
+
+	for (;;) {
+		u32 hdr, op, count;
+
+		if (off + 4 > len)
+			break;
+		memcpy(&hdr, buf + off, 4);
+		if (hdr == 0)
+			break;	/* trailing zero padding */
+		op = (hdr >> 29) & 0x7;
+		count = (hdr >> 16) & 0xfff;
+		switch (op) {
+		case 1:		/* INC_METHOD */
+		case 3:		/* NON_INC_METHOD */
+		case 5:		/* ONE_INC */
+			if (!count || count > 64 ||
+			    off + 4 + count * 4 > len)
+				return false;
+			off += 4 + count * 4;
+			break;
+		case 4:		/* IMMD_DATA_METHOD: header only */
+			off += 4;
+			break;
+		case 7:		/* END_PB_SEGMENT: header only */
+			off += 4;
+			break;
+		default:
+			return false;
+		}
+		if (++methods > 1024)
+			return false;
+	}
+	while (off < len && buf[off] == 0)
+		off++;
+	return methods > 0 && off == len;
+}
+
+/*
+ * Does this buffer read as a push? The discovered VA->PA map is
  * content-scanned and can hand a push VA an unrelated page (observed: GSP
- * firmware and kernel data pages). A real push starts with a method at
- * address 0 (SET_OBJECT / INC_METHOD class bind) with a count that fits
- * the buffer.
+ * firmware and kernel data pages), so a readable page is only trusted when
+ * the whole entry parses as a method stream.
  */
 static bool svc_method_stream_plausible(const u8 *buf, u32 len)
 {
-	u32 hdr = 0, op, count;
-
-	if (len < 8)
-		return false;
-	memcpy(&hdr, buf, 4);
-	op = (hdr >> 29) & 0x7;
-	count = (hdr >> 16) & 0xfff;
-	return op == 1 && count && count <= 64 &&
-	       4 + count * 4 <= len && ((hdr & 0x1fff) << 2) == 0;
+	return svc_stream_parse(buf, len);
 }
 
 /*
@@ -2821,12 +2878,32 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			continue;
 		}
 
-		if (op != 1 || !count || count > 64 ||
+		if (op == 4 || op == 7) {
+			/* Header-only methods (immediate data in the header
+			 * for op 4; end-of-push segment for op 7). */
+			off += 4;
+			continue;
+		}
+
+		if ((op != 1 && op != 3 && op != 5) || !count || count > 64 ||
 		    off + 4 + count * 4 > len) {
 			off += 4;	/* resync */
 			continue;
 		}
 
+		if (op == 3 || op == 5) {
+			/*
+			 * NON_INC / ONE_INC methods carry their own data
+			 * words. Skip the method whole: resyncing word by
+			 * word through the data misreads payload words as
+			 * headers (observed: a NON_INC block whose data
+			 * held INC-shaped words queued a bogus LAUNCH_DMA).
+			 * No release is decoded from these encodings.
+			 */
+			off += 4 + count * 4;
+			continue;
+		}
+
 		if (count >= 1)
 			memcpy(&v0, buf + off + 4, 4);
 		if (count >= 2)
@@ -4486,27 +4563,7 @@ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot,
  */
 static bool svc_stream_full_ok(const u8 *buf, u32 len)
 {
-	u32 off = 0, methods = 0;
-
-	while (off + 4 <= len) {
-		u32 hdr, op, count;
-
-		memcpy(&hdr, buf + off, 4);
-		if (hdr == 0)
-			break;	/* trailing zero padding */
-		op = (hdr >> 29) & 0x7;
-		count = (hdr >> 16) & 0xfff;
-		if (op != 1 || !count || count > 64 ||
-		    off + 4 + count * 4 > len)
-			return false;
-		off += 4 + count * 4;
-		if (++methods > 1024)
-			return false;
-	}
-	/* Trailing zero padding is common in recycled push slots. */
-	while (off < len && buf[off] == 0)
-		off++;
-	return methods > 0 && off == len;
+	return svc_stream_parse(buf, len);
 }
 
 /*
@@ -5015,6 +5072,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			u64 va, idx = ch->last_get;
 			u32 len, e0, e1, g;
 
+			/*
+			 * Remember the payload this entry should deliver the
+			 * first time it is examined. If the pool advances past
+			 * it later, the release was already delivered (the
+			 * content sweep applies releases from RAM pages before
+			 * the pump reads their entries) and the entry must be
+			 * consumed instead of waiting out the grace window -
+			 * otherwise every push costs one full grace period.
+			 */
+			if (!ch->defer_since)
+				ch->entry_want = ch->expected_sem_payload;
+
 			if (rrc) {
 				stalled = true;	/* ring unreadable: sweep RAM */
 				break;
@@ -5158,6 +5227,27 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								if (!rel_seen && gsp_sema_fake_rel)
 									svc_fake_release(vgpu, ch);
 								else if (!rel_seen) {
+									/*
+									 * Already delivered by the
+									 * content sweep: the pool
+									 * advanced past this entry's
+									 * payload while it was pending
+									 * (or decoded no release). The
+									 * entry has nothing left to
+									 * deliver; consume it now, do
+									 * not pay the grace window.
+									 */
+									if (ch->entry_want &&
+									    ch->expected_sem_payload >
+									    ch->entry_want) {
+										ch->defer_since = 0;
+										ch->rescue_fail = 0;
+										ch->last_get = (idx + 1) %
+											ch->gpfifo_entries;
+										svc.gpfifo_skipped++;
+										kfree(pb);
+										continue;
+									}
 									if (!ch->defer_since)
 										ch->defer_since =
 											jiffies;
@@ -5326,6 +5416,27 @@ if (gsp_sema_fake_rel &&
 							 * - that consumed the client's
 							 * never-delivered wait.
 							 */
+							if (ch->entry_want &&
+							    ch->expected_sem_payload >
+							    ch->entry_want) {
+								/*
+								 * Already delivered
+								 * (the content sweep
+								 * applied this entry's
+								 * release from RAM):
+								 * consume now instead
+								 * of waiting the grace
+								 * window per push.
+								 */
+								ch->rescue_fail = 0;
+								ch->defer_since = 0;
+								ch->last_get =
+									(idx + 1) %
+									ch->gpfifo_entries;
+								svc.gpfifo_skipped++;
+								kfree(pb);
+								continue;
+							}
 							if (!ch->defer_since)
 								ch->defer_since =
 									jiffies;