vgsp: fail4 r242-r248 - GRCE LOAD_INLINE_DATA cursor advance + H2D decode instruments

The probe/mallocecho H2D (cudaMemcpy H2D of a 64 KB device buffer)
arrives as LOAD_INLINE_DATA method runs (op=3, subch=1, addr 0x1b4)
carrying the payload inline in the push, split across several runs
(and, depending on pool state, across several pushes of one sequence).
The inline-write handler wrote every run at st.offset_out without
advancing, so a multi-run upload kept only the last run's words and
the readback page held deterministic junk (RAWA 16382 16383 8193...).

Fix: a LOAD_INLINE_DATA run advances the destination cursor by the
bytes it wrote (hardware FIFO semantics); consecutive runs of one
upload now tile from the first destination.

Verified (fresh boot, standard params): mallocecho d-afterH2D head
0 1 2 3 4 5 6 7 (was junk); ctaid_probe RAWA first8 0 1 2 3 4 5 6 7
(was 16382 16383 8193 8194...). Residual: the upload spans multiple
runs/pushes and only the first 4095-word run lands today (bad=12289,
tail zeros) - the remaining runs are still lost to the decoder's
run-boundary handling (see sema-next-hdr captures).

Instruments added (all gsp_sema_dbg-gated, bounded, session-reset):
sema-pushall (de-windowed push decode trace with page_pa),
sema-opadd, sema-copyent, sema-dmatrace (denied transfers),
sema-bigrun (count>=1024 runs on len>=8192 pushes with cursor state),
sema-lid-hdr, sema-skiprun, sema-mystery, sema-next-hdr (header after
each big LID run). LDA/xfer-raw/stream dumps de-gated from
ch->h_object!=0 so unidentified channels are visible.

1 files changed, 159 insertions, 5 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 1e4ce61..a5b1d93 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -817,6 +817,16 @@ struct sema_svc {
 	u64			push_pat_log;		/* push-content pattern budget */
 	u64			dma_skip_log;		/* unrecognized-DMA budget */
 	u64			xfer_raw_log;		/* operandless transfer raw-dump budget */
+	u64			pushtrace_all;		/* fail4: de-windowed push decode
+						 * trace (every push, bounded) */
+	u64			opadd_log;		/* fail4: every op appended */
+	u64			dmatrace_log;		/* fail4: denied-transfer trace */
+	u64			copyent_log;		/* fail4: svc_copy entry trace */
+	u64			bigrun_log;		/* fail4: big method-data run headers */
+	u64			lid_hdr_log;		/* fail4: LOAD_INLINE_DATA headers */
+	u64			skiprun_log;		/* fail4: silently skipped data runs */
+	u64			mystery_log;		/* fail4: moderate op=1 runs on big pushes */
+	u64			next_hdr_log;		/* fail4: header after each big LID run */
 	u64			grce_log;		/* GRCE inline-write budget */
 	u32			cur_ch;			/* channel of the op being run */
 	u64			entry_log;		/* per-push summary budget */
@@ -5209,6 +5219,17 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		*done_out = 0;	/* 0 until the first byte lands */
 	if (!total || total > (256ULL << 20))
 		return -EINVAL;
+	/*
+	 * fail4: name every copy the executor is asked to run (pre-
+	 * resolution), so a copy that fails or is held before any witness
+	 * still has an entry record.
+	 */
+	if (gsp_sema_dbg && svc.copyent_log < 128) {
+		svc.copyent_log++;
+		pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n",
+			svc.cur_ch, dst, src, total, dst_phys, dst_aperture,
+			src_phys, src_aperture);
+	}
 
 	if (!dst_phys) {
 		u64 pa;
@@ -7391,7 +7412,21 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			ch->h_object, ch->engine_type, push_va, len,
 			allow_ops, trusted);
 	}
-	if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 96) {
+	/*
+	 * fail4: de-windowed decode trace - every push the service looks at,
+	 * bounded per session. The windowed/identified-channel diagnostics
+	 * above can miss a whole client's push stream (unidentified channel,
+	 * no bridge completion to arm the window); this record cannot.
+	 */
+	if (svc.pushtrace_all < 512) {
+		svc.pushtrace_all++;
+		pr_info("sema-pushall: h=0x%x eng=%u va=0x%llx pa=0x%llx len=%u allow=%d trusted=%d first8=%08x %08x\n",
+			ch->h_object, ch->engine_type, push_va, page_pa, len,
+			allow_ops, trusted,
+			len >= 4 ? *(const u32 *)(buf + 0) : 0,
+			len >= 8 ? *(const u32 *)(buf + 4) : 0);
+	}
+	if (gsp_sema_dbg && svc_dbg_streams < 96) {
 		svc_dbg_streams++;
 		pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n",
 			ch->h_object, ch->engine_type, push_va, len, allow_ops,
@@ -7439,6 +7474,30 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		addr = (hdr & 0x1fff) << 2;
 
 		/*
+		 * fail4: name every big method-data run and every run whose
+		 * address is the LOAD_INLINE_DATA method (any op/subch - the
+		 * encoding of the 64 KB H2D upload is what we are pinning).
+		 */
+		if (gsp_sema_dbg && len >= 8192 && count >= 1024 &&
+		    svc.bigrun_log < 128) {
+			svc.bigrun_log++;
+			pr_info("sema-bigrun: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x have_out=%u out=0x%llx line=%u lcnt=%u w=%08x %08x\n",
+				ch->h_object, push_va, off, hdr, op, count, subch,
+				addr, st.have_out, st.offset_out, st.line_length,
+				st.gr_line_count,
+				off + 8 <= len ? *(const u32 *)(buf + off + 4) : 0,
+				off + 12 <= len ? *(const u32 *)(buf + off + 8) : 0);
+		}
+		if (gsp_sema_dbg && len >= 8192 &&
+		    (addr == GRCE_M_LOAD_INLINE_DATA || addr == 0x1b8) &&
+		    svc.lid_hdr_log < 64) {
+			svc.lid_hdr_log++;
+			pr_info("sema-lid-hdr: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x trusted=%d allow=%d\n",
+				ch->h_object, push_va, off, hdr, op, count, subch,
+				addr, trusted, allow_ops);
+		}
+
+		/*
 		 * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries
 		 * the DMA source bytes in the push itself. Run the write to
 		 * the destination the preceding methods selected.
@@ -7456,7 +7515,35 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				svc_grce_inline_write(vgpu, st.offset_out,
 						      buf + off + 4, bytes,
 						      ch->h_object);
-			} else if (gsp_sema_dbg && svc.grce_log < 16) {
+				/*
+				 * fail4: hardware semantics - a LOAD_INLINE_DATA
+				 * run advances the destination cursor by the
+				 * bytes it wrote. Without this, the consecutive
+				 * runs of a split 64 KB upload all land at the
+				 * first destination and the page ends up with
+				 * only the last run's words (the RAWA junk).
+				 */
+				st.offset_out += bytes;
+				/*
+				 * fail4: name the header that follows each big
+				 * LID run - the split 64 KB upload's other
+				 * payload carriers sit exactly there.
+				 */
+				if (gsp_sema_dbg && len >= 4096 &&
+				    svc.next_hdr_log < 24) {
+					u32 noff = off + 4 + count * 4;
+					u32 nh = 0, nw = 0;
+
+					if (noff + 4 <= len)
+						memcpy(&nh, buf + noff, 4);
+					if (noff + 8 <= len)
+						memcpy(&nw, buf + noff + 4, 4);
+					svc.next_hdr_log++;
+					pr_info("sema-next-hdr: h=0x%x push=0x%llx after=0x%x cnt=%u nxt_hdr=0x%08x nxt_w0=%08x cursor=0x%llx\n",
+						ch->h_object, push_va, off,
+						count, nh, nw, st.offset_out);
+				}
+			} else if (gsp_sema_dbg && svc.grce_log < 64) {
 				svc.grce_log++;
 				pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n",
 					trusted, allow_ops, st.have_out,
@@ -7485,6 +7572,23 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			continue;
 		}
 
+		/*
+		 * fail4: name the moderate op=1 method runs on big pushes -
+		 * the split 64 KB upload's payload carriers beyond the LID
+		 * class land here (their addr selects the write port).
+		 */
+		if (gsp_sema_dbg && len >= 4096 && op == 1 &&
+		    count >= 16 && count <= 1023 && svc.mystery_log < 48) {
+			svc.mystery_log++;
+			pr_info("sema-mystery: h=0x%x push=0x%llx off=0x%x hdr=0x%08x cnt=%u subch=%u addr=0x%x have_out=%u out=0x%llx line=%u lcnt=%u w=%08x %08x %08x\n",
+				ch->h_object, push_va, off, hdr, count, subch,
+				addr, st.have_out, st.offset_out,
+				st.line_length, st.gr_line_count,
+				off + 8 <= len ? *(const u32 *)(buf + off + 4) : 0,
+				off + 12 <= len ? *(const u32 *)(buf + off + 8) : 0,
+				off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0);
+		}
+
 		if (op == 4 || op == 7) {
 			/* Header-only methods (immediate data in the header
 			 * for op 4; end-of-push segment for op 7). */
@@ -7549,6 +7653,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			 * held INC-shaped words queued a bogus LAUNCH_DMA).
 			 * No release is decoded from these encodings.
 			 */
+			if (gsp_sema_dbg && len >= 8192 && count >= 1024 &&
+			    svc.skiprun_log < 64) {
+				svc.skiprun_log++;
+				pr_info("sema-skiprun: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x trusted=%d allow=%d\n",
+					ch->h_object, push_va, off, hdr, op,
+					count, subch, addr, trusted, allow_ops);
+			}
 			off += 4 + count * 4;
 			continue;
 		}
@@ -7914,7 +8025,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * operand state, so one run covers both the
 				 * C5B5 and the GRCE copy shapes.
 				 */
-				if (ch->h_object && gsp_sema_dbg &&
+				if (gsp_sema_dbg &&
 				    svc.xfer_raw_log < 48) {
 					u32 dl = len < 192 ? len : 192;
 
@@ -7977,8 +8088,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		case CE_M_LAUNCH_DMA: {
 			u32 dma = v0;
 			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);
+			u32 n_ops_before = n_ops;
 
-			if (gsp_sema_dbg && ch->h_object &&
+			if (gsp_sema_dbg &&
 			    svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) {
 				svc.lda_log++;
 				pr_info("sema-lda: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n",
@@ -8019,7 +8131,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			 * bounded number of such launches so the operand
 			 * encoding (or its absence) is on record.
 			 */
-			if (transfer != 0 && ch->h_object && gsp_sema_dbg &&
+			if (transfer != 0 && gsp_sema_dbg &&
 			    svc.xfer_raw_log < 48 &&
 			    (!st.have_out || !st.line_length)) {
 				u32 dl = len < 192 ? len : 192;
@@ -8097,6 +8209,39 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			}
 
 			/*
+			 * fail4: name a transfer-shaped launch that decoded NO
+			 * op because the pass was denied (allow_ops=0) - the
+			 * op-chain above (and its dma-skip dump) is skipped
+			 * wholesale in that case, so this class was invisible.
+			 */
+			if (gsp_sema_dbg && transfer != 0 && !allow_ops &&
+			    svc.dmatrace_log < 64) {
+				svc.dmatrace_log++;
+				pr_info("sema-dmatrace: denied h=0x%x push=0x%llx dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u trusted=%d\n",
+					ch->h_object, push_va, dma,
+					st.line_length, st.offset_in,
+					st.offset_out, st.have_in, st.have_out,
+					st.have_remap, trusted);
+			}
+			/*
+			 * fail4: name every op appended to this pass (copy or
+			 * fill) with its resolved operand fields, so the
+			 * executor's population is on record even when the
+			 * 64 KB copy witness below never fires for it.
+			 */
+			if (n_ops != n_ops_before && svc.opadd_log < 256) {
+				struct svc_memset_op *lm = &ops[n_ops - 1];
+
+				svc.opadd_log++;
+				pr_info("sema-opadd: h=0x%x type=%u dst=0x%llx src=0x%llx bytes=%llu elem=%u count=%u ap=%u src_ap=%u phys=%u src_phys=%u\n",
+					ch->h_object, lm->type, lm->dst,
+					lm->src, lm->bytes, lm->elem,
+					lm->count, lm->aperture,
+					lm->src_aperture, lm->phys,
+					lm->src_phys);
+			}
+
+			/*
 			 * Both release encodings write the payload word first
 			 * (NvGpuSemaphore.data[0] for the four-word format, the
 			 * single word for the one-word format), and the client's
@@ -8784,6 +8929,15 @@ static void svc_session_flush(void)
 	svc.counter_noslot_dbg = 0;
 	svc.sem256_log = 0;
 	svc.xfer_raw_log = 0;
+	svc.pushtrace_all = 0;
+	svc.opadd_log = 0;
+	svc.dmatrace_log = 0;
+	svc.copyent_log = 0;
+	svc.bigrun_log = 0;
+	svc.lid_hdr_log = 0;
+	svc.skiprun_log = 0;
+	svc.mystery_log = 0;
+	svc.next_hdr_log = 0;
 	/*
 	 * The push decode budget (`sema-push` lines) is per-session too: the
 	 * first client of a boot otherwise spends it and the next session's