vgsp: fail4 - denied device-dst copies hold+retry (fix the chunk drop)

Root cause of the vector_add chunk drop (CONT.55 chase): the sw-copy
client's GR-classified GRCE channel (eng=0, h=0x5c000059) decodes its
1MB H2D chunk copies as CE transfers, but ops on that channel run only
once the pin/trust stream (injected) reaches each push page. The first
pass over a chunk push lands before that; with allow_ops=0 the copy was
never built and the entry was consumed silently. All later passes
classify as replays (slot_ops_complete=1), so the entry was never
retried: 6 of 8 chunks skipped, kernel computed on stale VRAM
(vector_add r302-r307: 2-5 of 8 chunk ops ever existed, the rest never
ran; the +2MB 'phase' signature is stale residue).

Fix: a denied launch that WOULD have built a type=1 device-dst copy
(no remap, both offsets, out >= SVC_VAS_FLOOR) with a trusted push now
marks the pass op-pending. op_tr_pending un-completes the slot, so the
next pass is treated fresh and the copy runs as soon as trust (injected
with allow=1) or the operand translation arrives - the existing
m163/opdef hold+retry machinery. Replay passes are deliberately
eligible (that is what makes the retry possible); a per-slot
slot_pend_tries cap (4 per put epoch) prevents a slot whose trust never
arrives from wedging the channel in 10s hold cycles.

Measured (r308, sema_dbg): all 8 consecutive chunk dsts build ops
(0x...cc..d3) and retry to success - 28 CEAUDIT rc=0 soft copies, 38
rc=0 vs 20 rc=-2 executions; previously max 5 dsts and only the last
2-3 ran. Regression checks on r309 with the fix: probe RAWA2+PROBE2B
bad=0, victory_mail OK, victory_matmul OK (one transient first-run
fail, clean re-run), transformer OK. vector_add now advances through
the full H2D set but still does not complete (retry storms stretch the
copy phase; the run later wedges in the GR stale-wedge recovery loop -
named as the next lead).

Diagnostics kept (dbg-gated, bounded): sema-pend set/hold witness,
sema-consume consumed-slot witness, sema-entry de-gated for eng=0
channels, dmatrace extended with replay/eng/pend_would, budgets raised
under sema_dbg so boot init cannot spend the trace before the run.

1 files changed, 106 insertions, 9 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b884bb5..4259314 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -260,6 +260,8 @@ struct svc_channel {
 				 * window (observed: one 8 s grace per push,
 				 * serializing the runtime init) */
 	DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS);
+	u8	slot_pend_tries[SVC_GPFIFO_MAX_BITS];	/* fail4 r308: bounded
+				* re-holds of a denied device-dst copy slot */
 				/* GPFIFO slots already consumed in the current
 				 * put epoch (the client's put value since it
 				 * last wrapped). A pin-batch rewind replays
@@ -821,6 +823,7 @@ struct sema_svc {
 						 * trace (every push, bounded) */
 	u64			opadd_log;		/* fail4: every op appended */
 	u64			dmatrace_log;		/* fail4: denied-transfer trace */
+	u64			pend_dbg_log;		/* fail4 r305: pending/hold witness */
 	u64			copyent_log;		/* fail4: svc_copy entry trace */
 	u64			bigrun_log;		/* fail4: big method-data run headers */
 	u64			lid_hdr_log;		/* fail4: LOAD_INLINE_DATA headers */
@@ -6359,7 +6362,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 		 * read-back signature: always keep room for it past the
 		 * general trace budget so the DtoH's src/dst/rc is on record.
 		 */
-		if ((svc.op_trace_log < 2048 &&
+		if ((svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) &&
 		     (time_before(jiffies, svc.op_trace_until) ||
 		      m->dst >= SVC_VAS_FLOOR ||
 		      (m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) ||
@@ -7579,7 +7582,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	svc_fwd_defer_armed = false;
 	svc.pages_decoded++;
 	/* Diagnostic: name every push decoded in the post-completion window. */
-	if (svc.op_trace_log < 2048 &&
+	if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) &&
 	    time_before(jiffies, svc.op_trace_until)) {
 		svc.op_trace_log++;
 		pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n",
@@ -7592,7 +7595,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	 * above can miss a whole client's push stream (unidentified channel,
 	 * no bridge completion to arm the window); this record cannot.
 	 */
-	if (svc.pushtrace_all < 512) {
+	/*
+	 * fail4 r302: with sema_dbg on the boot-time driver init spends the
+	 * 512-line budget long before the run under diagnosis, leaving the
+	 * interesting push stream (a later client's copies) invisible. The
+	 * debug flag raises the bound so one boot covers init + run.
+	 */
+	if (svc.pushtrace_all < (gsp_sema_dbg ? 32768 : 512)) {
 		svc.pushtrace_all++;
 		pr_info("sema-pushall: h=0x%x eng=%u va=0x%llx pa=0x%llx len=%u allow=%d trusted=%d first8=%08x %08x\n",
 			ch->h_object, ch->engine_type, push_va, page_pa, len,
@@ -8333,7 +8342,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			}
 			/* Diagnostic: full operand state in the post-completion
 			 * window (or for high device VAs), bounded. */
-			if (svc.op_trace_log < 2048 && ch->h_object &&
+			if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && ch->h_object &&
 			    (time_before(jiffies, svc.op_trace_until) ||
 			     st.offset_out >= SVC_VAS_FLOOR ||
 			     st.offset_in >= SVC_VAS_FLOOR)) {
@@ -8441,13 +8450,79 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			 * wholesale in that case, so this class was invisible.
 			 */
 			if (gsp_sema_dbg && transfer != 0 && !allow_ops &&
-			    svc.dmatrace_log < 64) {
+			    svc.dmatrace_log < (gsp_sema_dbg ? 4096 : 64)) {
 				svc.dmatrace_log++;
-				pr_info("sema-dmatrace: denied h=0x%x push=0x%llx dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u trusted=%d\n",
+				pr_info("sema-dmatrace: denied h=0x%x push=0x%llx dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u trusted=%d replay=%d eng=%u pend_would=%d\n",
 					ch->h_object, push_va, dma,
 					st.line_length, st.offset_in,
 					st.offset_out, st.have_in, st.have_out,
-					st.have_remap, trusted);
+					st.have_remap, trusted, replay,
+					ch->engine_type,
+					(!allow_ops && trusted &&
+					 !ch->engine_type && transfer != 0 &&
+					 st.line_length && st.have_out &&
+					 st.have_in &&
+					 !DMA_REMAP_ENABLE(dma) &&
+					 st.offset_out >= SVC_VAS_FLOOR &&
+					 entry_idx != SVC_ENTRY_NONE &&
+					 entry_idx < SVC_GPFIFO_MAX_BITS &&
+					 ch->slot_pend_tries[entry_idx] < 4) ? 1 : 0);
+			}
+			/*
+			 * fail4 r302: a denied device-destination copy that
+			 * WOULD have built a type=1 op (the exact branch shape
+			 * above: no remap, both offsets, device dst) must not
+			 * be consumed silently. This is the sw-copy client
+			 * mode: the GR-classified GRCE channel's ops run only
+			 * once the pin/trust stream (injected) reaches the
+			 * entry, and the first pass over a chunk push lands
+			 * before that - the copy then never ran and the guest
+			 * computed on stale VRAM (vector_add r302: chunk
+			 * copies cc..d1 skipped; only the tail entries were
+			 * re-decoded by a rewind and ran). Mark the pass
+			 * op-pending so the pump holds the entry through the
+			 * m163/opdef machinery and the retry runs the copy as
+			 * soon as the trust (or the operand translation)
+			 * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps
+			 * the pre-existing skip as the last resort.
+			 */
+			if (!allow_ops && trusted &&
+			    !ch->engine_type && transfer != 0 &&
+			    st.line_length && st.have_out && st.have_in &&
+			    !DMA_REMAP_ENABLE(dma) &&
+			    st.offset_out >= SVC_VAS_FLOOR) {
+				/*
+				 * r308: the `!replay` gate above kept this
+				 * dead on the sw-copy channel: after the first
+				 * consumed pass every later pass classifies as
+				 * a replay, so the denied chunk copies were
+				 * never retried (vector_add: 6 of 8 chunks
+				 * skipped). A replay pass that marks the slot
+				 * pending un-completes it (slot_ops_complete
+				 * is written from svc.op_tr_pending below), so
+				 * the next pass is treated fresh and the copy
+				 * runs once trust/translation arrives. The
+				 * per-slot counter caps the re-holds so a slot
+				 * whose trust never arrives cannot wedge the
+				 * channel in 10 s hold cycles.
+				 */
+				if (entry_idx != SVC_ENTRY_NONE &&
+				    entry_idx < SVC_GPFIFO_MAX_BITS &&
+				    ch->slot_pend_tries[entry_idx] < 4) {
+					ch->slot_pend_tries[entry_idx]++;
+					svc.op_tr_pending = 1;
+					if (gsp_sema_dbg &&
+					    svc.pend_dbg_log < 512) {
+						svc.pend_dbg_log++;
+						pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n",
+							ch->h_object, push_va,
+							entry_idx, st.offset_out,
+							st.offset_in,
+							st.line_length,
+							replay, trusted,
+							ch->slot_pend_tries[entry_idx]);
+					}
+				}
 			}
 			/*
 			 * fail4: name every op appended to this pass (copy or
@@ -8938,7 +9013,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		off += 4 + count * 4;
 	}
 
-	if (gsp_sema_dbg && ch->h_object && svc.entry_log < 512) {
+	if (gsp_sema_dbg && ch->h_object &&
+	    svc.entry_log < 16384) {
 		svc.entry_log++;
 		{
 			u8 tail[24];
@@ -12993,6 +13069,20 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu,
 /* Mark a consumed GPFIFO slot for the current put epoch. */
 static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put)
 {
+	static u32 consume_log;
+
+	/*
+	 * fail4 r303: name every consumed slot (all channels). Paired with
+	 * the sema-entry decode rows this shows exactly which entries were
+	 * consumed without ops and when (the vector_add chunk-drop chase:
+	 * a consumed slot whose decode row carried a device-dst copy but
+	 * whose op never ran is the silent skip under diagnosis).
+	 */
+	if (gsp_sema_dbg && consume_log < 16384) {
+		consume_log++;
+		pr_info("sema-consume: h=0x%x idx=%u put=%u\n",
+			ch->h_object, idx, put);
+	}
 	(void)put;
 	/*
 	 * STALL FIX (entry=82/pay=84 class, r186): the entry leaving the
@@ -17082,7 +17172,7 @@ static bool svc_push_read_displaced(struct nv_vgpu_device *vgpu, u32 h,
 static void svc_entry_outcome(const char *what, const struct svc_channel *ch,
 			      u64 idx, u64 va, u32 len)
 {
-	if (!gsp_sema_dbg || !ch->engine_type || svc.gpfifo_entry_log >= 512)
+	if (!gsp_sema_dbg || svc.gpfifo_entry_log >= 16384)
 		return;
 	svc.gpfifo_entry_log++;
 	pr_info("sema-entry: %s h=0x%x idx=%llu va=0x%llx len=%u want=%u exp=%u unread=%d\n",
@@ -17781,6 +17871,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				}
 			}
 			bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS);
+			memset(ch->slot_pend_tries, 0,
+			       sizeof(ch->slot_pend_tries));
 			/* GRFIX FIX1b: a recycled slot in the next put epoch
 			 * gets a fresh absolute-bound record; without this a
 			 * wrapped slot would inherit a stale expired launch
@@ -18410,6 +18502,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							    svc_opdef_hold(vgpu, ch,
 									   (u32)idx)) {
 								svc.gpfifo_nomap++;
+								if (gsp_sema_dbg && svc.pend_dbg_log < 1024) {
+									svc.pend_dbg_log++;
+									pr_info("sema-pend: hold h=0x%x idx=%u va=0x%llx\n",
+										ch->h_object, (u32)idx, va);
+								}
 								/*
 								 * P6b wave 1.5:
 								 * releases recorded