vgsp-sema: SESSION 126 FENCE FOLD - channel-anchored sweep of a GR channel's PRE-BUILT ring continuation beyond GP_PUT (sema_gr_ahead, default 64): the live Vulkan client's fence waits never clear because the driver's kick boundary stops at the first batch (put=11) while the ring already carries the continuation entries (12..47) whose INC-5 RELEASE groups target the fence words - r716 ring dumps: beec0100 idx17 0x49035b0 len=1020 (group at +0x374 -> 0x45ba080), idx25/33/41/46 at 0x4903f5c/0x4904594/0x4904bb4/0x49051d4 (len 136/112 -> 0x45ba090), all beyond the GP_PUT-bounded walk; the sweep walks entries beyond put through the ring's validated GPFIFO translation, accepts only entries passing the strict method-stream gate (all fence entries parse), extracts addr-0x5c count>=5 RELEASE/REDUCTION groups and queues them forward-only via the standard pending ledger - no consumption bookkeeping touched, idempotent, gated by gr_pub_armed (live epoch) and rrc==0; also: sema-pushdec budget 512->2048/client (boot-time driver-init reads exhausted it before the client's first submit on r716) and chan_dump gains armed=%d

1 files changed, 173 insertions, 2 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index c50832e..13ad648 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -23671,6 +23671,165 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu,
 			    put, get, ch->submissions, waited, reason);
 }
 
+/*
+ * ---------------------------------------------------------------------
+ * SESSION 126 - FENCE FOLD: channel-anchored sweep of a GR-class
+ * channel's PRE-BUILT ring continuation.
+ *
+ * The Vulkan client's fence waits never clear unaided because the
+ * driver's kick boundary (GP_PUT) stops at the first batch while the ring
+ * already carries the continuation entries it pre-built for the next
+ * submits - and the fence's own INC-5 RELEASE group (observed
+ * [2005c017][0x45ba090][0][payload 1][0][0x02100001]) lives in one of
+ * those entries, so the pump's GP_PUT-bounded walk never decodes it
+ * (r715/r716: put=11, entries 12..47 carried content, no dmesg row ever
+ * mentioned 0x45ba). The host-side fenceword_assist wrote the wait word
+ * directly; this fold makes the service author it from the driver's own
+ * push bytes.
+ *
+ * Doctrine: unlike the disabled page-content sweep (svc_apply_page_data,
+ * off since the false-positive corruption incidents), this sweep is
+ * CHANNEL-ANCHORED: it only walks entries reachable through the ring's
+ * validated GPFIFO translation, only accepts entries whose bytes pass the
+ * strict method-stream gate, and only applies the semaphore release
+ * groups it finds - forward-only, through the standard pending ledger
+ * (svc_pending_add). No consumption bookkeeping is touched (last_get,
+ * put_prev, slot_done_epoch and the replay state stay as the pump left
+ * them), so a later real kick of the same entries decodes them normally;
+ * repeated sweeps are idempotent (the pending ledger dedups (va,payload)
+ * and the apply path is forward-only).
+ *
+ * Gated by the same live-epoch arm as the counter publish
+ * (ch->gr_pub_armed) so a residual ring from a previous session cannot
+ * author releases before the session switch is proven.
+ * ---------------------------------------------------------------------
+ */
+unsigned gsp_sema_gr_ahead_max = 64;
+module_param_named(sema_gr_ahead, gsp_sema_gr_ahead_max, uint, 0644);
+MODULE_PARM_DESC(sema_gr_ahead,
+	"GR-class channels: max GPFIFO entries beyond GP_PUT whose channel-anchored pushes are swept for semaphore releases (0=off)");
+
+static u32 svc_gr_ahead_log;
+static u64 svc_gr_ahead_entries;
+static u64 svc_gr_ahead_rels;
+static u64 svc_gr_ahead_skips;
+
+/*
+ * Extract the semaphore release groups from one coherent push and queue
+ * them forward-only. Mirrors the decode sites' group shape exactly:
+ * addr 0x5c, count >= 5, op RELEASE (1) or REDUCTION (6); the address is
+ * w0/w1 as a 40-bit VA and the payload is w2.
+ */
+static u32 svc_gr_ahead_push(struct svc_channel *ch, const u8 *buf, u32 len)
+{
+	u32 off = 0, nrel = 0;
+
+	while (off + 4 <= len) {
+		u32 hdr, op, addr, count;
+
+		memcpy(&hdr, buf + off, 4);
+		if (!hdr) {
+			off += 4;
+			continue;
+		}
+		op = (hdr >> 29) & 0x7;
+		addr = (hdr & 0x1fff) << 2;
+		count = (hdr >> 16) & 0xfff;
+		if (op == 0 || op == 2 || op == 4 || op == 7) {
+			off += 4;
+			continue;
+		}
+		if (op != 1 && op != 3 && op != 5) {
+			off += 4;
+			continue;
+		}
+		if (!count || count > 0xfff || off + 4 + count * 4 > len) {
+			off += 4;
+			continue;
+		}
+		if (addr == HOST_M_SEM_ADDR_LO && count >= 5) {
+			u32 w0, w1, w2, w4;
+			u64 va;
+
+			memcpy(&w0, buf + off + 4, 4);
+			memcpy(&w1, buf + off + 8, 4);
+			memcpy(&w2, buf + off + 12, 4);
+			memcpy(&w4, buf + off + 20, 4);
+			va = ((u64)(w1 & 0xff) << 32) |
+			     (u64)(w0 & 0xfffffffc);
+			op = w4 & 0x7;
+			if ((op == HOST_SEM_OP_RELEASE ||
+			     op == HOST_SEM_OP_REDUCTION) && w2 &&
+			    va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_HI) {
+				svc_pending_add(va, w2,
+					op == HOST_SEM_OP_REDUCTION ? 1 : 0);
+				nrel++;
+			}
+		}
+		off += 4 + count * 4;
+	}
+	(void)ch;
+	return nrel;
+}
+
+static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu,
+			       struct svc_channel *ch, u32 put)
+{
+	static u8 abuf[2048];
+	u32 i, maxent, nrel = 0, swept = 0, nskip = 0;
+
+	if (!put || !ch->gpfifo_entries)
+		return;
+	maxent = min_t(u32, gsp_sema_gr_ahead_max, 512);
+	for (i = 0; i < maxent; i++) {
+		u32 idx = (put + i) % ch->gpfifo_entries;
+		u8 ent[SVC_GPFIFO_ENTRY_SIZE];
+		u32 e0, e1, len, rlen;
+		u64 va;
+
+		if (svc_read_va(vgpu,
+				ch->gpfifo_va +
+				(u64)idx * SVC_GPFIFO_ENTRY_SIZE,
+				ent, sizeof(ent)))
+			break;
+		memcpy(&e0, ent, 4);
+		memcpy(&e1, ent + 4, 4);
+		va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+		len = ((e1 >> 10) & 0x1fffff) << 2;
+		if (!va && !len)
+			break;	/* unwritten slot: end of the continuation */
+		if (!va || !len || len > SVC_PUSH_MAX)
+			break;
+		if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI)
+			break;
+		rlen = min_t(u32, len, sizeof(abuf));
+		if (svc_read_va_push(vgpu, ch, va, abuf, rlen)) {
+			/*
+			 * Unreadable here (pre-anchor translation) is a
+			 * reason to stop this pass: the continuation is not
+			 * proven coherent, so nothing is authored.
+			 */
+			nskip++;
+			break;
+		}
+		if (!svc_stream_full_ok(abuf, rlen)) {
+			nskip++;
+			continue;
+		}
+		swept++;
+		nrel += svc_gr_ahead_push(ch, abuf, rlen);
+	}
+	svc_gr_ahead_entries += swept;
+	svc_gr_ahead_rels += nrel;
+	svc_gr_ahead_skips += nskip;
+	if (swept && (nrel || svc_gr_ahead_log < 64)) {
+		svc_gr_ahead_log++;
+		pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u skipped=%u (total rels=%llu)\n",
+			ch->h_object, put, swept, nrel, nskip,
+			svc_gr_ahead_rels);
+	}
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	if (svc_pdb_disc_pending) {
@@ -24198,6 +24357,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		ch->put_prev = put;
 
 		/*
+		 * SESSION 126 fence fold: sweep the ring continuation beyond
+		 * GP_PUT for semaphore releases (see svc_gr_ahead_sweep).
+		 * GR-class channels only, live-armed epochs only, and only
+		 * with the ring translation proven through the channel
+		 * object (rrc == 0).
+		 */
+		if (!ch->engine_type && ch->gr_pub_armed &&
+		    gsp_sema_gr_ahead_max && !rrc && ring_pa)
+			svc_gr_ahead_sweep(vgpu, ch, put);
+
+		/*
 		 * STALLFREE r225: stale-epoch / stale-replay wedge watchdog.
 		 * Runs before the entry loop so a wedged session epoch is
 		 * recovered (and logged) instead of spinning: see
@@ -24695,7 +24865,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					 * Budget: 512 rows per client (reset in
 					 * svc_session_flush).
 					 */
-					if (gsp_sema_dbg && svc_pushdec_log < 512) {
+					if (gsp_sema_dbg && svc_pushdec_log < 2048) {
 						svc_pushdec_log++;
 						pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n",
 							ch->h_object, va, len,
@@ -26847,12 +27017,13 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu)
 			svc_read_ap(vgpu, ch->notifier_pa + 16 + 8,
 				    ch->notifier_as, &info1, 4);
 		}
-		pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n",
+		pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx armed=%d notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n",
 			ch->h_object, ch->engine_type, ch->gpfifo_va,
 			ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
 			ch->userd_ok, ch->last_get, ch->submissions,
 			ch->expected_sem_payload, ch->last_rel_va,
 			ch->counter_pa,
+			ch->gr_pub_armed,
 			ch->notifier_pa,
 			ch->notifier_as, info0, st0, info1, st1,
 			ch->notified);