gr: single scheduling edge (pcas default 0) - fixes double engine execution; retire-wait tracked-total fix; sema acquire wait gate per-entry + C56F op semantics (opt-in); fill-shape decode diagnostic

1 files changed, 358 insertions, 73 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index fbb278d..8623f07 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3550,6 +3550,10 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
 }
 
 static void svc_pending_add(u64 va, u32 payload, u32 op);
+/* r34x: defined with the pending ring / opdef slots (later in the file);
+ * the acquire gate drains/appraises withheld releases before it holds. */
+static void svc_apply_pending(struct nv_vgpu_device *vgpu);
+static bool svc_wait_pend_release(u64 va);
 
 /*
  * F1: GR release-word authorship checks. Defined next to the svc_fwd_req
@@ -3965,22 +3969,59 @@ static u64 svc_ledger_diverges;
 static u64 svc_ledger_pinwins;		/* r336: pin-backed divergence wins */
 static u32 svc_d2hmap_log;		/* D2H-MAP per-page destination budget */
 /*
- * r336: host-semaphore acquire (C56F SEM_EXECUTE op 0/2/3/4/5) wait state.
- * The decoder records the last acquire of the current push; svc_run_ops
- * honors it before running any of the push's work, so a copy cannot
- * execute ahead of the dependency its push waits on (the streams2
- * readback-before-scale class). The re-decode re-parses the push, so the
- * state is refreshed on every pass.
- */
-static u64 svc_wait_va;
-static u32 svc_wait_payload;
-static u8 svc_wait_mode;		/* 0 = equal, 1 = geq, 2 = circ geq */
-static u8 svc_wait_pending;
+ * r336/r34x: host-semaphore acquire (C56F SEM_EXECUTE op 0/2/3/4/5) wait
+ * state. The decoder captures EVERY acquire of the ring entry currently
+ * being decoded (svc_waits[], reset per entry at the top of
+ * svc_decode_stream), and svc_run_ops honors them before running any of
+ * the entry's work, so a copy cannot execute ahead of the dependency its
+ * push waits on (the streams2 readback-before-scale class). Per-entry
+ * capture is essential: the wait state used to leak across entries (a
+ * later launch's self-release wait, e.g. need=85 while the held push
+ * releases 84, gated an earlier entry that carries no acquire at all -
+ * the r340 over-hold). The re-decode re-parses the entry, so the state
+ * is refreshed on every pass.
+ *
+ * Semantics per op (class contract, see svc_wait_satisfied):
+ *   0 ACQUIRE = (v == p); 2 ACQ_STRICT_GEQ = v >= p; 3 ACQ_CIRC_GEQ =
+ *   circular (s32)(v - p) >= 0; 4 ACQ_AND = (v & p) != 0;
+ *   5 ACQ_NOR = ~(v | p) != 0.
+ */
+#define SVC_WAIT_MAX		8
+#define SVC_WAIT_HOLD_MS	100u	/* bounded acquire wait (fail-open) */
+struct svc_wait_op {
+	u64	va;
+	u32	payload;
+	u8	op;		/* C56F SEM_EXECUTE operation */
+	u32	off;		/* method offset in the entry (order) */
+};
+static struct svc_wait_op svc_waits[SVC_WAIT_MAX];
+static u32 svc_wait_n;			/* acquires captured this entry */
+static u32 svc_wait_ovf;		/* acquires dropped past the array */
 static u64 svc_wait_held;
 static u64 svc_wait_pass;
+static u64 svc_wait_giveup;
 static u32 svc_wait_log;		/* per-session held-row print budget */
-static u64 svc_wait_rel_va;		/* current push's own release (mirror) */
+static u32 svc_wait_pass_log;		/* per-session resolved-row budget */
+static u32 svc_remapconst_log;		/* r343 FILL: SET_REMAP_CONST_A rows */
+static u32 svc_fill_noremap;		/* r343 FILL: fail-closed holds */
+static u64 svc_wait_rel_va;		/* this entry's own release (mirror) */
 static u32 svc_wait_rel_payload;
+static u32 svc_wait_rel_off;		/* its method offset (order check) */
+static bool svc_wait_rel_have;
+/*
+ * Per-channel first-hold clock for the bounded wait: a gate evaluation
+ * that cannot satisfy the acquire arms a 100 ms budget per hold episode;
+ * expiry fails open (ops run) with a loud 'give-up' row. The table is
+ * tiny (a handful of channels), service-thread only, cleared per pass
+ * resolution / session flush.
+ */
+struct svc_wait_hold {
+	bool		active;
+	u32		h;
+	unsigned long	since;
+};
+#define SVC_WAIT_HOLD_MAX 16
+static struct svc_wait_hold svc_wait_holds[SVC_WAIT_HOLD_MAX];
 static u64 svc_ledger_stores;
 static u64 svc_ledger_evictions;
 static u64 svc_ledger_evict_since_reset;
@@ -4108,11 +4149,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.gr_nocap_held, svc.gr_nocap_consumed,
 		svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel,
 		svc.gr_stale_wedge, svc_pending_ovf);
-	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu\n",
+	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n",
 		svc.counter_pub_calls, svc.gr_rel_applied,
 		svc.sem_write_reresolve, svc.pushpg_fallback,
 		svc.pool_alt_applied, svc.pool_residue,
-		svc_wait_held, svc_wait_pass);
+		svc_wait_held, svc_wait_pass, svc_wait_giveup,
+		svc_wait_ovf);
 	if (svc.resc_run || svc_rescue_scan_capped)
 		pr_info("sema-svc:   rescue_run=%llu scan_capped=%llu (multi-page content pins)\n",
 			svc.resc_run, svc_rescue_scan_capped);
@@ -6278,14 +6320,60 @@ static void svc_handle_map_add(void);
 /* Execute a decoded operation list (fills and copies). `ops_done` (may be
  * NULL) is the per-entry bitmap of ops that already ran for the entry's
  * identity; `ops_base` is the sequence number of ops[0] within the entry. */
-/* r336: evaluate a C56F SEM_EXECUTE acquire against a candidate value. */
-static bool svc_wait_satisfied(u32 cur, u8 mode, u32 need)
+
+/*
+ * r34x: evaluate a C56F SEM_EXECUTE acquire against the semaphore word.
+ * The class defines the opcodes (clc56f.h NVC56F_SEM_EXECUTE_OPERATION:
+ * 0 ACQUIRE, 1 RELEASE, 2 ACQ_STRICT_GEQ, 3 ACQ_CIRC_GEQ, 4 ACQ_AND,
+ * 5 ACQ_NOR, 6 REDUCTION) and the CUDA driver documents the wait
+ * conditions each op must implement (cuda_device.h CUIsemaphoreAquire-
+ * Flags_enum: GEQ default = circular >=, EQUAL = ==, AND = (v & p) != 0,
+ * NOR = ~(v | p) != 0, STRICT_GEQ = >= with no wrap; ampere_semaphore.c
+ * maps those flags onto the opcodes: EQUAL->0, STRICT_GEQ->2, GEQ->3,
+ * AND->4, NOR->5). Note op 0 is EQUALITY, not a >=: only op 2/3 are
+ * ordering compares. 32-bit payloads: every observed CUDA acquire is
+ * PAYLOAD_SIZE 32BIT (SEM_PAYLOAD_HI == 0); a 64-bit acquire is captured
+ * and judged on the low word only (its high word is not modelled).
+ */
+static bool svc_wait_satisfied(u32 cur, u8 op, u32 need)
 {
-	if (mode == 0)
+	switch (op) {
+	case 0:	/* ACQUIRE: wait for v == p */
 		return cur == need;
-	if (mode == 2)
+	case 2:	/* ACQ_STRICT_GEQ: v >= p, no wrap */
+		return cur >= need;
+	case 3:	/* ACQ_CIRC_GEQ: v in [p, p + 2^31 - 1] (wrap allowed) */
 		return (s32)(cur - need) >= 0;
-	return cur >= need;
+	case 4:	/* ACQ_AND: (v & p) != 0 */
+		return (cur & need) != 0;
+	case 5:	/* ACQ_NOR: ~(v | p) != 0 */
+		return ~(cur | need) != 0;
+	default:
+		return true;
+	}
+}
+
+/* r34x: the per-channel first-hold clock for the bounded wait. */
+static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create)
+{
+	struct svc_wait_hold *free_slot = NULL;
+	int i;
+
+	for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) {
+		struct svc_wait_hold *e = &svc_wait_holds[i];
+
+		if (e->active && e->h == h)
+			return e;
+		if (!e->active && !free_slot)
+			free_slot = e;
+	}
+	if (!create || !free_slot)
+		return NULL;
+	memset(free_slot, 0, sizeof(*free_slot));
+	free_slot->active = true;
+	free_slot->h = h;
+	free_slot->since = jiffies;
+	return free_slot;
 }
 
 static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
@@ -6307,16 +6395,34 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 	 */
 	svc_handle_map_add();
 	/*
-	 * r336: honor the push's host-semaphore acquire (C56F SEM_EXECUTE
-	 * op 0/2/3/4/5) before any of its work runs. The push's release is
-	 * withheld through the op-pending machinery (bounded, loud), so the
-	 * guest cannot observe completion of work that ran ahead of the
-	 * dependency it waits on. Unreadable semaphore: fail-open (the
-	 * pre-r336 behaviour).
+	 * r336/r34x: honor every host-semaphore acquire (C56F SEM_EXECUTE
+	 * op 0/2/3/4/5) captured for THIS ring entry before any of its work
+	 * runs, so a copy cannot execute ahead of the dependency its push
+	 * waits on (the streams2 readback-before-scale class). Decision
+	 * ladder per wait:
+	 *   1. saved value satisfies the op's condition -> pass;
+	 *   2. the entry's own release, when its method offset precedes the
+	 *      acquire's (release-then-acquire, channel.c crossEngineSync),
+	 *      counts as the candidate value - the release method has
+	 *      already executed by the time the acquire is evaluated;
+	 *      an acquire that precedes its own trailing tracking release
+	 *      does NOT count it (nothing has installed that value yet);
+	 *   3. drain the pending-translation release queue (releases OUR
+	 *      machinery is merely late about) and re-read once;
+	 *   4. still unsatisfied -> hold the entry (release withheld through
+	 *      the op-pending machinery) for at most SVC_WAIT_HOLD_MS per
+	 *      episode, then fail open loudly ('give-up': the value cannot
+	 *      be installed by firmware alone - it sits behind an upstream
+	 *      opdef/operand hold - so the old pass-through behaviour is
+	 *      safer than parking the channel forever). Unreadable
+	 *      semaphore: fail-open (the pre-r336 behaviour).
+	 * The pass/hold/give-up decisions are logged as 'sema-wait' rows
+	 * (op, payload, observed value, decision).
 	 */
-	if (svc_wait_pending && gsp_sema_wait_gate) {
-		u32 cur = 0;
-		bool ok = true;
+	if (svc_wait_n && gsp_sema_wait_gate) {
+		struct svc_wait_hold *gh;
+		bool hold = false;
+		u32 wi;
 
 		/*
 		 * Kill switch: a wait that never satisfies (a counter our
@@ -6334,33 +6440,80 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 			svc_wait_pass++;
 			goto svc_wait_done;
 		}
-		if (!svc_sem_read(vgpu, svc_wait_va, &cur))
-			ok = svc_wait_satisfied(cur, svc_wait_mode,
-						svc_wait_payload);
-		/*
-		 * The driver's release(N)+acquire(N) pattern (channel.c
-		 * crossEngineSync): the push's own release lands only after
-		 * its ops run, so it cannot satisfy the wait through memory
-		 * yet - treat its payload as applied.
-		 */
-		if (!ok && svc_wait_rel_va == svc_wait_va &&
-		    svc_wait_rel_payload)
-			ok = svc_wait_satisfied(svc_wait_rel_payload,
-						svc_wait_mode,
-						svc_wait_payload);
-		if (!ok) {
+		gh = svc_wait_hold_get(ch_h, true);
+		for (wi = 0; wi < svc_wait_n; wi++) {
+			const struct svc_wait_op *w = &svc_waits[wi];
+			u32 cur = 0, waited_ms = 0;
+			bool ok = true, same_rel = false, pend = false;
+
+			if (gh)
+				waited_ms = jiffies_to_msecs(jiffies - gh->since);
+			if (svc_sem_read(vgpu, w->va, &cur))
+				continue;	/* unreadable: fail-open */
+			ok = svc_wait_satisfied(cur, w->op, w->payload);
+			if (!ok && svc_wait_rel_have &&
+			    svc_wait_rel_va == w->va &&
+			    svc_wait_rel_off < w->off) {
+				same_rel = true;
+				ok = svc_wait_satisfied(svc_wait_rel_payload,
+							w->op, w->payload);
+			}
+			if (!ok) {
+				svc_apply_pending(vgpu);
+				if (!svc_sem_read(vgpu, w->va, &cur))
+					ok = svc_wait_satisfied(cur, w->op,
+								w->payload);
+			}
+			if (ok) {
+				if (svc_wait_pass_log < 256) {
+					svc_wait_pass_log++;
+					pr_info("sema-wait: pass ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u waited_ms=%u -> ops run this pass\n",
+						ch_h, w->va, w->op, w->payload,
+						cur, same_rel, waited_ms);
+				}
+				continue;
+			}
+			pend = svc_wait_pend_release(w->va);
+			if (waited_ms >= SVC_WAIT_HOLD_MS) {
+				/*
+				 * Bounded episode expired: fail open for the
+				 * WHOLE entry (one row, then the ops run). A
+				 * per-wait continue here would ping-pong
+				 * with a second unsatisfied wait: the first
+				 * gave up and reset the shared clock before
+				 * the second was judged, and the pair would
+				 * hold forever 100 ms at a time.
+				 */
+				svc_wait_giveup++;
+				if (svc_wait_log < 2048) {
+					svc_wait_log++;
+					pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n",
+						ch_h, w->va, w->op, w->payload,
+						cur, pend, waited_ms);
+				}
+				if (gh)
+					gh->active = false;
+				hold = false;
+				svc_wait_pass++;
+				goto svc_wait_done;
+			}
+			hold = true;
+			if (svc_wait_log < 512) {
+				svc_wait_log++;
+				pr_info("sema-wait: hold ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u pend=%u waited_ms=%u -> entry held (acquire not satisfied)\n",
+					ch_h, w->va, w->op, w->payload, cur,
+					same_rel, pend, waited_ms);
+			}
+			break;	/* one unsatisfied wait holds the entry */
+		}
+		if (hold) {
 			svc_wait_held++;
-			if (svc_wait_log < 512)
-				pr_info("sema-wait: held ch=0x%x r=%u va=0x%llx need=%u cur=%u mode=%u rel=0x%llx rpay=%u (acquire not satisfied)\n",
-					ch_h, svc_wait_log++,
-					svc_wait_va, svc_wait_payload,
-					cur, svc_wait_mode,
-					svc_wait_rel_va,
-					svc_wait_rel_payload);
 			svc.op_tr_pending = 1;
 			return;
 		}
 		svc_wait_pass++;
+		if (gh)
+			gh->active = false;
 	}
 svc_wait_done:
 	for (i = 0; i < n_ops; i++) {
@@ -7687,6 +7840,25 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	 */
 	svc.cur_ch = ch->h_object;
 	/*
+	 * r34x: the acquire-wait capture is PER RING ENTRY. It used to be
+	 * reset only at SET_OBJECT, so a decode pass over a later entry (or
+	 * a set of entries, since a held entry is re-decoded after the
+	 * sweep walks ahead) leaked its acquire into an earlier entry's
+	 * release-site gate: r340 held a D2H copy entry that carries no
+	 * acquire at all on a later launch's 'wait for the next tracking
+	 * value' (need=85 while the held entry releases 84) - a circular
+	 * wait only our own withheld release could ever satisfy, so every
+	 * hold expired via the 10 s fallback (wait=4542 etc.). Reset here,
+	 * per entry; SET_OBJECT still resets at the push boundary, and the
+	 * re-decode re-captures the same waits, so held entries re-evaluate
+	 * identically.
+	 */
+	svc_wait_n = 0;
+	svc_wait_rel_va = 0;
+	svc_wait_rel_payload = 0;
+	svc_wait_rel_off = 0;
+	svc_wait_rel_have = false;
+	/*
 	 * Static for the same reason as svc_handle_chan_dump: a 32-entry ops
 	 * array is a ~2 KB frame, and this is called from the single service
 	 * thread only.
@@ -8118,9 +8290,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			/* SET_OBJECT on the channel subchannel: a push starts */
 			n_ops = 0;
 			memset(&st, 0, sizeof(st));
-			svc_wait_pending = 0;	/* r336: new push, new wait */
+			/* r336/r34x: new push, new acquire set + release mirror */
+			svc_wait_n = 0;
 			svc_wait_rel_va = 0;
 			svc_wait_rel_payload = 0;
+			svc_wait_rel_off = 0;
+			svc_wait_rel_have = false;
 			off += 4 + count * 4;
 			continue;
 		}
@@ -8141,6 +8316,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				st.rel_payload = payload;
 				svc_wait_rel_va = va;
 				svc_wait_rel_payload = payload;
+				svc_wait_rel_off = off;
+				svc_wait_rel_have = true;
 				st.have_rel = 1;
 				svc_rel_va_seen = va;
 				svc_rel_count++;
@@ -8242,24 +8419,39 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			} else if (op != HOST_SEM_OP_RELEASE &&
 				   va >= SEMA_VA_LO && va < SEMA_VA_HI) {
 				/*
-				 * r336: C56F SEM_EXECUTE acquire ops (0 =
-				 * acquire/equal, 2 = strict GEQ, 3 = circular
-				 * GEQ, 4/5 = AND/NOR). Core CUDA starts its
-				 * compute-channel pushes with one of these to
-				 * wait for a progress counter (observed:
+				 * r336/r34x: C56F SEM_EXECUTE acquire ops
+				 * (0 = ACQUIRE/equal, 2 = strict GEQ,
+				 * 3 = circular GEQ, 4 = AND, 5 = NOR).
+				 * Core CUDA starts its compute-channel
+				 * pushes with one of these to wait for a
+				 * progress counter (observed:
 				 * [LO=0x0400ffd0][HI=2][pay=4][0][0x1003] on
-				 * the D2H readback push - ACQ_CIRC_GEQ on the
-				 * channel counter 0x20400ffd0). svc_run_ops
-				 * honors the wait before running the push's
-				 * work, so a copy cannot execute ahead of the
+				 * the D2H readback push - ACQ_CIRC_GEQ on
+				 * the channel counter 0x20400ffd0).
+				 * svc_run_ops honors every captured acquire
+				 * of this entry before running its work, so
+				 * a copy cannot execute ahead of the
 				 * dependency it waits on (the streams2
-				 * readback-before-scale class).
+				 * readback-before-scale class). Capturing
+				 * into the per-entry list (instead of a
+				 * single last-wait global) is what stops a
+				 * later entry's acquire from gating an
+				 * earlier entry (the r340 over-hold).
+				 * PAYLOAD_SIZE 64BIT (execute bit 24) is
+				 * captured on the low word only; every
+				 * observed CUDA acquire is 32-bit.
 				 */
-				svc_wait_va = va;
-				svc_wait_payload = payload;
-				svc_wait_mode = (op == 3) ? 2 :
-						(op == 0) ? 0 : 1;
-				svc_wait_pending = 1;
+				if (svc_wait_n < SVC_WAIT_MAX) {
+					struct svc_wait_op *w =
+						&svc_waits[svc_wait_n++];
+
+					w->va = va;
+					w->payload = payload;
+					w->op = (u8)op;
+					w->off = off;
+				} else {
+					svc_wait_ovf++;
+				}
 			}
 			off += 4 + count * 4;
 			continue;
@@ -8311,6 +8503,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					st.rel_payload = st.rs_payload;
 					svc_wait_rel_va = va;
 					svc_wait_rel_payload = st.rs_payload;
+					svc_wait_rel_off = off;
+					svc_wait_rel_have = true;
 					st.have_rel = 1;
 					svc_rel_va_seen = va;
 					svc_rel_count++;
@@ -8488,6 +8682,24 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			st.have_sem = 1;
 			break;
 		case CE_M_SET_REMAP_CONST_A:
+			/*
+			 * r343 FILL instrument: a fill-shaped LAUNCH_DMA
+			 * (REMAP_ENABLE + destination PITCH, OFFSET_OUT and
+			 * no OFFSET_IN) needs this constant to build its op.
+			 * The 8MB-memset class (memcpy_roundtrip all-bad,
+			 * victory_memset2d) shows fill-shaped launches with
+			 * have_remap=0 that decode nothing and are consumed
+			 * silently. Log every decode (bounded) so one run
+			 * captures the real run shape even when the count
+			 * guard below cannot use it.
+			 */
+			if (svc_remapconst_log < 32) {
+				svc_remapconst_log++;
+				pr_info("sema-remapconst: h=0x%x push=0x%llx off=0x%x count=%u v0=0x%x v1=0x%x v2=0x%x trusted=%d allow=%u\n",
+					ch->h_object, push_va, off, count,
+					v0, count >= 2 ? v1 : 0u,
+					count >= 3 ? v2 : 0u, trusted, allow_ops);
+			}
 			if (count >= 3) {
 				st.remap_a = v0;
 				st.remap_b = v1;
@@ -8630,6 +8842,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			}
 
 			/*
+			 * r343 FILL diagnostic (LOG-ONLY, no hold): a
+			 * fill-shaped transfer (REMAP_ENABLE + destination
+			 * PITCH, OFFSET_OUT and no OFFSET_IN) whose value
+			 * constant did not decode currently decodes no op
+			 * and is consumed silently - the guest's cudaMemset
+			 * then reports success with zero bytes written
+			 * (r341 memcpy_roundtrip: 8MB memset readback
+			 * all-bad; victory_memset2d). An op_tr_pending hold
+			 * here WEDGED client init (r343: first cudaMalloc
+			 * retried forever - driver init issues fill-shaped
+			 * CE pushes, the hold re-armed every decode pass
+			 * and never expired). So: log loudly only; the
+			 * sema-remapconst rows above capture the real
+			 * decode shape to build the proper fill decoder.
+			 */
+			if (allow_ops && transfer != 0 &&
+			    st.line_length >= 1024 && st.have_out &&
+			    !st.have_in && DMA_REMAP_ENABLE(dma) &&
+			    DMA_DST_MEMORY_LAYOUT_PITCH(dma) &&
+			    !st.have_remap && n_ops == n_ops_before &&
+			    svc_fill_noremap < 8) {
+				svc_fill_noremap++;
+				pr_err("vgsp: FILL-NOREMAP h=0x%x push=0x%llx dma=0x%08x line=%u out=0x%llx: fill-shaped launch, value source not decoded (log-only diagnostic)\n",
+				       ch->h_object, push_va, dma,
+				       st.line_length, st.offset_out);
+			}
+
+			/*
 			 * fail4: name a transfer-shaped launch that decoded NO
 			 * op because the pass was denied (allow_ops=0) - the
 			 * op-chain above (and its dma-skip dump) is skipped
@@ -8866,6 +9106,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				st.rel_payload = st.sem_payload;
 				svc_wait_rel_va = sem_va;
 				svc_wait_rel_payload = st.sem_payload;
+				svc_wait_rel_off = off;
+				svc_wait_rel_have = true;
 				st.have_rel = 1;
 
 				if (st.have_sem && st.sem_payload &&
@@ -9607,6 +9849,12 @@ static void svc_session_flush(void)
 	svc.dmatrace_log = 0;
 	svc.copyent_log = 0;
 	svc_wait_log = 0;
+	svc_wait_pass_log = 0;
+	svc_remapconst_log = 0;
+	svc_fill_noremap = 0;
+	svc_wait_n = 0;
+	svc_wait_rel_have = false;
+	memset(svc_wait_holds, 0, sizeof(svc_wait_holds));
 	svc.bigrun_log = 0;
 	svc.lid_hdr_log = 0;
 	svc.skiprun_log = 0;
@@ -11823,13 +12071,19 @@ bool gsp_sema_pool_hold_fresh = 1;
  */
 bool gsp_sema_copy_dst_pin_guard = 1;
 /*
- * r336 wait gate: honor the C56F host-semaphore acquire before running a
- * push's work. DEFAULT OFF: the acquire's tracking-semaphore values are
- * not yet fully modelled (channel-first pushes over-hold: the wait needs a
- * counter value only the withheld release itself would install), so the
- * gate stays an experiment until the counter semantics are finished. The
- * streams2 readback-before-scale class is root-caused in the handoff;
- * with the gate off the behaviour is the pre-r336 one.
+ * r336/r34x wait gate: honor the C56F host-semaphore acquire ops captured
+ * for a ring entry before running its work. Still DEFAULT OFF (opt-in
+ * experiment; the module param sema_wait_gate=1 enables it), but the
+ * semantics are now the class contract's: op 0 = equal, 2 = strict GEQ,
+ * 3 = circular GEQ, 4 = AND, 5 = NOR; the wait set is captured PER ENTRY
+ * (the old single-last-wait global leaked a later entry's acquire into an
+ * earlier entry - the r340 over-hold); the entry's own release counts only
+ * when its method order precedes the acquire; pending releases our
+ * machinery is merely late about are drained first; and every hold is
+ * bounded at SVC_WAIT_HOLD_MS (100 ms) then fails open loudly ('give-up')
+ * so a value that can only be installed behind an upstream opdef hold
+ * cannot park a channel. Diagnostics: 'sema-wait: pass/hold/give-up' rows
+ * carry op, payload, observed value and decision.
  */
 bool gsp_sema_wait_gate;
 
@@ -14130,6 +14384,37 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu,
 }
 
 /*
+ * r34x: is a release to this VA currently withheld by our own machinery?
+ * True when the pending-translation ring holds one (translation late) or
+ * an opdef slot has one recorded (operand hold). The acquire gate calls
+ * this to classify an unsatisfied wait in its diagnostics: pend=1 names
+ * the 'value sits behind our own withhold' case - the bounded 100 ms
+ * give-up is exactly for it (waiting longer cannot install the value
+ * until the upstream entry's ops complete). Never forces an opdef release
+ * here: those releases are withheld because their ops have NOT completed,
+ * and delivering early re-opens the work-before-fence hole they close.
+ */
+static bool svc_wait_pend_release(u64 va)
+{
+	u32 i;
+
+	for (i = 0; i < svc_pending_n; i++)
+		if (svc_pending[i].va == va)
+			return true;
+	for (i = 0; i < SVC_OPDEF_SLOTS; i++) {
+		struct svc_opdef_ent *e = &svc_opdef[i];
+		u8 k;
+
+		if (!e->active)
+			continue;
+		for (k = 0; k < e->n; k++)
+			if (e->rel[k].va == va)
+				return true;
+	}
+	return false;
+}
+
+/*
  * STALL FIX (r186 entry=82/pay=84): called from svc_replay_note() on every
  * entry-consume path. If a release set was withheld for this entry while an
  * operand was pending and the entry is now leaving the decode path (so the