vgsp: C56F acquire wait gate as an opt-in experiment (default off)

The streams2 readback-before-scale class is root-caused: core CUDA
starts compute-channel pushes with a C56F SEM_EXECUTE acquire
(ACQ_CIRC_GEQ on a tracking semaphore; e.g. the D2H readback push
waits on 0x20400fef0 >= its sequence number) and the decoder ignored
it, so the copy ran ahead of the scale launch it waits for. The gate
implemented here (decode the acquire; svc_run_ops holds the push's
work until the semaphore satisfies, with the driver's
release(N)+acquire(N) same-push pattern treated as satisfied;
op-pending bounded hold; kill switch at 2048 holds; diagnostics
sema-wait held rows with a per-session budget) OVER-HOLDS on boots
where a channel's first pushes wait on counter values only their own
withheld release would install (wait counts reach thousands, the
client stalls; r337/r338/r339). The tracking-semaphore value
semantics (what the guest word should read vs the driver's sequence
numbers) need a dedicated pass before this can be enabled.

Default OFF so behaviour is the pre-r336 one; enable per-boot with
sema_wait_gate=1 for the experiment. Everything else (pin-wins,
vector_add, the battery) is unaffected.

3 files changed, 72 insertions, 13 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index e8a6a14..fbb278d 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3978,6 +3978,9 @@ static u8 svc_wait_mode;		/* 0 = equal, 1 = geq, 2 = circ geq */
 static u8 svc_wait_pending;
 static u64 svc_wait_held;
 static u64 svc_wait_pass;
+static u32 svc_wait_log;		/* per-session held-row print budget */
+static u64 svc_wait_rel_va;		/* current push's own release (mirror) */
+static u32 svc_wait_rel_payload;
 static u64 svc_ledger_stores;
 static u64 svc_ledger_evictions;
 static u64 svc_ledger_evict_since_reset;
@@ -6275,6 +6278,16 @@ static void svc_handle_map_add(void);
 /* Execute a decoded operation list (fills and copies). `ops_done` (may be
  * NULL) is the per-entry bitmap of ops that already ran for the entry's
  * identity; `ops_base` is the sequence number of ops[0] within the entry. */
+/* r336: evaluate a C56F SEM_EXECUTE acquire against a candidate value. */
+static bool svc_wait_satisfied(u32 cur, u8 mode, u32 need)
+{
+	if (mode == 0)
+		return cur == need;
+	if (mode == 2)
+		return (s32)(cur - need) >= 0;
+	return cur >= need;
+}
+
 static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 			u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base)
 {
@@ -6301,30 +6314,55 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 	 * dependency it waits on. Unreadable semaphore: fail-open (the
 	 * pre-r336 behaviour).
 	 */
-	if (svc_wait_pending) {
+	if (svc_wait_pending && gsp_sema_wait_gate) {
 		u32 cur = 0;
 		bool ok = true;
 
-		if (!svc_sem_read(vgpu, svc_wait_va, &cur)) {
-			if (svc_wait_mode == 0)
-				ok = (cur == svc_wait_payload);
-			else if (svc_wait_mode == 2)
-				ok = (s32)(cur - svc_wait_payload) >= 0;
-			else
-				ok = cur >= svc_wait_payload;
+		/*
+		 * Kill switch: a wait that never satisfies (a counter our
+		 * emulation does not advance the way the driver expects)
+		 * would otherwise park entries until the sweep expires each
+		 * one. After 2048 holds in a boot the gate fails open
+		 * permanently (the pre-r336 behaviour) and says so once.
+		 */
+		if (svc_wait_held > 2048) {
+			if (svc_wait_log < 4096) {
+				svc_wait_log++;
+				pr_info("sema-wait: gate disabled after %llu holds (fail-open)\n",
+					svc_wait_held);
+			}
+			svc_wait_pass++;
+			goto svc_wait_done;
 		}
+		if (!svc_sem_read(vgpu, svc_wait_va, &cur))
+			ok = svc_wait_satisfied(cur, svc_wait_mode,
+						svc_wait_payload);
+		/*
+		 * The driver's release(N)+acquire(N) pattern (channel.c
+		 * crossEngineSync): the push's own release lands only after
+		 * its ops run, so it cannot satisfy the wait through memory
+		 * yet - treat its payload as applied.
+		 */
+		if (!ok && svc_wait_rel_va == svc_wait_va &&
+		    svc_wait_rel_payload)
+			ok = svc_wait_satisfied(svc_wait_rel_payload,
+						svc_wait_mode,
+						svc_wait_payload);
 		if (!ok) {
 			svc_wait_held++;
-			if (svc_wait_held <= 32)
-				pr_info("sema-wait: held ch=0x%x va=0x%llx need=%u cur=%u mode=%u (acquire not satisfied)\n",
-					ch_h, svc_wait_va,
-					svc_wait_payload, cur,
-					svc_wait_mode);
+			if (svc_wait_log < 512)
+				pr_info("sema-wait: held ch=0x%x r=%u va=0x%llx need=%u cur=%u mode=%u rel=0x%llx rpay=%u (acquire not satisfied)\n",
+					ch_h, svc_wait_log++,
+					svc_wait_va, svc_wait_payload,
+					cur, svc_wait_mode,
+					svc_wait_rel_va,
+					svc_wait_rel_payload);
 			svc.op_tr_pending = 1;
 			return;
 		}
 		svc_wait_pass++;
 	}
+svc_wait_done:
 	for (i = 0; i < n_ops; i++) {
 		struct svc_memset_op *m = &ops[i];
 		u32 seq = ops_base + i;
@@ -8081,6 +8119,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			n_ops = 0;
 			memset(&st, 0, sizeof(st));
 			svc_wait_pending = 0;	/* r336: new push, new wait */
+			svc_wait_rel_va = 0;
+			svc_wait_rel_payload = 0;
 			off += 4 + count * 4;
 			continue;
 		}
@@ -8099,6 +8139,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				svc.pushes_seen++;
 				st.rel_va = va;
 				st.rel_payload = payload;
+				svc_wait_rel_va = va;
+				svc_wait_rel_payload = payload;
 				st.have_rel = 1;
 				svc_rel_va_seen = va;
 				svc_rel_count++;
@@ -8267,6 +8309,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					svc.pushes_seen++;
 					st.rel_va = va;
 					st.rel_payload = st.rs_payload;
+					svc_wait_rel_va = va;
+					svc_wait_rel_payload = st.rs_payload;
 					st.have_rel = 1;
 					svc_rel_va_seen = va;
 					svc_rel_count++;
@@ -8820,6 +8864,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 st.sem_va_lo;
 				st.rel_va = sem_va;
 				st.rel_payload = st.sem_payload;
+				svc_wait_rel_va = sem_va;
+				svc_wait_rel_payload = st.sem_payload;
 				st.have_rel = 1;
 
 				if (st.have_sem && st.sem_payload &&
@@ -9560,6 +9606,7 @@ static void svc_session_flush(void)
 	svc.opadd_log = 0;
 	svc.dmatrace_log = 0;
 	svc.copyent_log = 0;
+	svc_wait_log = 0;
 	svc.bigrun_log = 0;
 	svc.lid_hdr_log = 0;
 	svc.skiprun_log = 0;
@@ -11775,6 +11822,16 @@ bool gsp_sema_pool_hold_fresh = 1;
  * 0 restores the historical immediate-write behavior.
  */
 bool gsp_sema_copy_dst_pin_guard = 1;
+/*
+ * r336 wait gate: honor the C56F host-semaphore acquire before running a
+ * push's work. DEFAULT OFF: the acquire's tracking-semaphore values are
+ * not yet fully modelled (channel-first pushes over-hold: the wait needs a
+ * counter value only the withheld release itself would install), so the
+ * gate stays an experiment until the counter semantics are finished. The
+ * streams2 readback-before-scale class is root-caused in the handoff;
+ * with the gate off the behaviour is the pre-r336 one.
+ */
+bool gsp_sema_wait_gate;
 
 /*
  * m169b: the guard above must never deadlock the boot's first client. The
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 55d323c..6b411c9 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -302,6 +302,7 @@ extern bool gsp_sema_pool_hold_fresh;
  * walk/RAM scan until a pin confirms (default on).
  */
 extern bool gsp_sema_copy_dst_pin_guard;
+extern bool gsp_sema_wait_gate;
 
 /*
  * r231 READ-ORDER guard (default 0: witness only). Defer a device-src copy
diff --git a/kernel/main.c b/kernel/main.c
index c150a1a..47a1aba 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2824,6 +2824,7 @@ MODULE_PARM_DESC(sema_pool_hold_fresh, "apply pool releases at fresh-pin time in
  * the m163 retry path re-resolve with pins.
  */
 module_param_named(sema_copy_dst_pin_guard, gsp_sema_copy_dst_pin_guard, bool, 0644);
+module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644);
 MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)");
 
 /*