sema-svc: run CE work before its release, grace unreadable CE entries, payload-pattern audit

- CE LAUNCH_DMA releases are now applied after the push's collected work
  runs: the client reads copied data as soon as it observes the release,
  so release-first let cuMemcpyDtoH observe a copy that had not executed.
- Unreadable CE entries stay pending for a grace window (sema_ce_grace_ms,
  default 8s) before the release fallback consumes them, so a translation
  that lands within a service pass lets the push be decoded and its work
  executed instead of being dropped.
- sema_copy_pat matches an 8-byte value against every copy's src/dst first
  word and scans decoded pushes for it, naming the submission that carries
  a known payload regardless of size/encoding.
- sema-dma-skip logs transfer-shaped launches that collect no op; op log
  lines carry the channel handle; LAUNCH_DMA inventory follows OPS_LOG.
- A real release application now records the release VA, so decoded pushes
  no longer trigger a spurious fake release.

3 files changed, 163 insertions, 43 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index bafe5c1..9b03cc5 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -344,6 +344,10 @@ struct sema_svc {
 	u64			ring_ops;		/* op runs from ring-decoded pushes */
 	u64			lda_log;		/* LAUNCH_DMA inventory budget */
 	u64			copy_watch_log;		/* sema_copy_watch log budget */
+	u64			copy_pat_log;		/* sema_copy_pat log budget */
+	u64			push_pat_log;		/* push-content pattern budget */
+	u64			dma_skip_log;		/* unrecognized-DMA budget */
+	u32			cur_ch;			/* channel of the op being run */
 	u64			entry_log;		/* per-push summary budget */
 	u64			ops_denied;		/* work refused: implausible stream */
 	u64			false_trans;		/* resolvable VA mapped to non-push */
@@ -2024,9 +2028,12 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 	 * Data-path watch: for copies of exactly sema_copy_watch bytes, log
 	 * the first 8 source and destination bytes so a broken link in the
 	 * chain (wrong source page, wrong destination, silent wipe) is
-	 * visible directly in the log.
+	 * visible directly in the log. With sema_copy_pat set, any copy whose
+	 * source or destination starts with that 8-byte value is logged too,
+	 * whatever its size - this finds the push carrying a known payload.
 	 */
-	if (gsp_sema_copy_watch && total == gsp_sema_copy_watch) {
+	if ((gsp_sema_copy_watch && total == gsp_sema_copy_watch) ||
+	    gsp_sema_copy_pat) {
 		u8 sb[8] = {0}, db[8] = {0};
 
 		if (src_aperture == CE_APERTURE_VID) {
@@ -2045,7 +2052,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		} else {
 			svc_read(vgpu, dst, db, 8);
 		}
-		if (svc.copy_watch_log < 64) {
+		if (gsp_sema_copy_watch && total == gsp_sema_copy_watch &&
+		    svc.copy_watch_log < 64) {
 			svc.copy_watch_log++;
 			pr_info("sema-copy-watch: total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8_before=%02x %02x %02x %02x %02x %02x %02x %02x\n",
 				total, dst, dst_aperture, src, src_aperture,
@@ -2053,6 +2061,23 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 				sb[6], sb[7], db[0], db[1], db[2], db[3],
 				db[4], db[5], db[6], db[7]);
 		}
+		if (gsp_sema_copy_pat && svc.copy_pat_log < 64) {
+			u64 sv = 0, dv = 0;
+
+			memcpy(&sv, sb, 8);
+			memcpy(&dv, db, 8);
+			if (sv == gsp_sema_copy_pat || dv == gsp_sema_copy_pat) {
+				svc.copy_pat_log++;
+				pr_info("sema-copy-pat: %s h=0x%x total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8=%02x %02x %02x %02x %02x %02x %02x %02x\n",
+					sv == gsp_sema_copy_pat ? "SRC" : "DST",
+					svc.cur_ch, total, dst, dst_aperture,
+					src, src_aperture,
+					sb[0], sb[1], sb[2], sb[3], sb[4],
+					sb[5], sb[6], sb[7], db[0], db[1],
+					db[2], db[3], db[4], db[5], db[6],
+					db[7]);
+			}
+		}
 	}
 
 	for (o = 0; o < total; ) {
@@ -2110,10 +2135,11 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 
 /* Execute a decoded operation list (fills and copies). */
 static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
-			u32 n_ops)
+			u32 n_ops, u32 ch_h)
 {
 	u32 i;
 
+	svc.cur_ch = ch_h;
 	for (i = 0; i < n_ops; i++) {
 		struct svc_memset_op *m = &ops[i];
 		int rc;
@@ -2134,11 +2160,12 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 			svc.memsets_fail++;
 		if ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) ||
 		    (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 8))
-			pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d\n",
+			pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n",
 				m->type == 1 ? "memcopy" : "memset",
 				m->dst, m->value, m->elem, m->count,
 				m->bytes, m->phys, m->aperture, m->src,
-				m->src_phys, m->src_aperture, rc);
+				m->src_phys, m->src_aperture, rc,
+				svc.cur_ch);
 	}
 }
 
@@ -2470,6 +2497,29 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			len >= 32 ? *(const u32 *)(buf + 28) : 0);
 	}
 
+	/*
+	 * Payload watch: with sema_copy_pat set, scan the push bytes for the
+	 * 8-byte value and name the push that contains it (channel, VA, offset
+	 * in the push). This finds the submission carrying a known payload
+	 * even when its method encoding is not (yet) decoded into an op.
+	 */
+	if (gsp_sema_copy_pat && svc.push_pat_log < 32) {
+		u32 o;
+
+		for (o = 0; o + 8 <= len; o += 4) {
+			u64 v;
+
+			memcpy(&v, buf + o, 8);
+			if (v == gsp_sema_copy_pat) {
+				svc.push_pat_log++;
+				pr_info("sema-push-pat: h=0x%x eng=%u va=0x%llx len=%u off=0x%x allow=%d trusted=%d\n",
+					ch_h, ch_eng, push_va, len, o,
+					allow_ops, trusted);
+				break;
+			}
+		}
+	}
+
 	while (off + 4 <= len) {
 		u32 hdr = 0, op, count, subch, addr;
 		u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0;
@@ -2572,7 +2622,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				/* The push's work always runs; see the
 				 * REPORT_SEM comment below. */
 				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
-					svc_run_ops(vgpu, ops, n_ops);
+					svc_run_ops(vgpu, ops, n_ops, ch_h);
 					svc.ring_ops++;
 					ops_runs++;
 				}
@@ -2668,7 +2718,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 * cannot run swept-RAM false positives.
 					 */
 				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
-					svc_run_ops(vgpu, ops, n_ops);
+					svc_run_ops(vgpu, ops, n_ops, ch_h);
 					svc.ring_ops++;
 					ops_runs++;
 				}
@@ -2756,7 +2806,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			u32 dma = v0;
 			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);
 
-			if (ch_h && svc.lda_log < 48) {
+			if (ch_h && svc.lda_log <
+			    (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) {
 				svc.lda_log++;
 				pr_info("sema-lda: h=0x%x eng=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n",
 					ch_h, ch_eng, dma, transfer,
@@ -2807,6 +2858,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					m->src_aperture = st.have_src_mode ?
 						st.src_phys_mode : CE_APERTURE_VID;
 					n_ops++;
+				} else if (svc.dma_skip_log < 32) {
+					/*
+					 * A launch that looks like a transfer but
+					 * carries no branch we execute: no source
+					 * offset and no remap constant. Log the
+					 * operand state once to expose an
+					 * encoding we do not decode yet (a
+					 * missing OFFSET_IN run, or a phys-mode
+					 * value that never arrived).
+					 */
+					svc.dma_skip_log++;
+					pr_info("sema-dma-skip: h=0x%x dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u have_sem=%u\n",
+						ch_h, dma, st.line_length,
+						st.offset_in, st.offset_out,
+						st.have_in, st.have_out,
+						st.have_remap, st.have_sem);
 				}
 			}
 
@@ -2844,17 +2911,24 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				}
 
 				/*
-				 * Apply the release BEFORE running the push's
-				 * collected work. The client's tracker only
-				 * observes the release, and its context-init
-				 * deadline is what the whole run races against;
-				 * running the memsets first added their full
-				 * execution time to every push's visible
-				 * completion latency. Ordering is still safe:
-				 * this is the only writer of both the semaphore
-				 * and the vidmem shadow, and the next push is
-				 * not decoded until this call returns.
+				 * The push's collected work runs BEFORE the
+				 * release is applied: the client reads the
+				 * copied data as soon as it observes the
+				 * release, so a release-first order lets it
+				 * observe completion of a copy that has not
+				 * executed yet (observed: cuMemcpyDtoH
+				 * returning zeros). The work is small (the
+				 * payload copies, <= 4 KB each) and runs on
+				 * this thread before the release write, so
+				 * the visible completion latency grows by
+				 * only the copy itself.
 				 */
+				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
+					svc_run_ops(vgpu, ops, n_ops, ch_h);
+					svc.ring_ops++;
+					ops_runs++;
+				}
+
 				if (pending) {
 					svc.pushes_pending++;
 					if (DMA_SEM_REDUCTION_EN(dma) &&
@@ -2880,28 +2954,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						/* handled by the exact-slot path */
 					} else if (!svc_sem_write(vgpu, sem_va, cur)) {
 						svc.releases_applied++;
+						svc_rel_va_seen = sem_va;
 						svc.post_hint_until = jiffies + 10 * HZ;
 					}
 				}
 
 				/*
-				 * The push's collected work always runs on real
-				 * hardware; the release only orders observers.
-				 * Execute it here, independently of whether the
-				 * semaphore is translatable or still behind -
-				 * gating the work on the semaphore drops the
-				 * page-table writes that create those very
-				 * translations (UVM's CE memsets into vidmem).
-				 * Ops are only ever collected from pushes read
-				 * through a trusted ring translation.
-				 */
-				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
-					svc_run_ops(vgpu, ops, n_ops);
-					svc.ring_ops++;
-					ops_runs++;
-				}
-
-				/*
 				 * Exact-slot fallback: the releasing channel's
 				 * completion counter was enumerated from the
 				 * client's pagemap, so the release can be
@@ -3741,6 +3799,22 @@ unsigned gsp_sema_ops_log = 8;
 /* Data-path watch: copy size whose first bytes are logged (0 = off). */
 unsigned gsp_sema_copy_watch;
 
+/*
+ * Data-path watch: 8-byte value matched against a copy's first source and
+ * destination words (either end), logged regardless of size (0 = off). Used
+ * to find the push that carries a known payload when its size is unknown.
+ */
+u64 gsp_sema_copy_pat;
+
+/*
+ * Grace window (ms) an untranslatable CE entry stays pending before the
+ * fallback acknowledges its release and consumes it. The push's work (data
+ * copies the client will read back) must run before the release the client
+ * synchronizes on; the translation usually lands within a service pass, so
+ * keep the entry pending inside the window instead of dropping the work.
+ */
+unsigned gsp_sema_ce_grace_ms = 8000;
+
 static unsigned long svc_gr_ring_tried;	/* ranks that got a diagnostic line */
 
 static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
@@ -4487,16 +4561,37 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 						 */
 						svc_log_stalled_va(va);
 						/*
-						 * With the fallback enabled the rescue
-						 * is pointless for CE: it never
-						 * executes the work anyway (only the
-						 * release), and its multi-second RAM
-						 * scan is what made the driver's
-						 * progress waits expire. Acknowledge
-						 * the release and move on.
+						 * The push's work (the data copies
+						 * the client reads back) must run
+						 * before the release the client
+						 * synchronizes on: acknowledging the
+						 * release without executing the copy
+						 * lets the client read unstaged
+						 * memory (observed: cuMemcpyDtoH
+						 * returning zeros after its HtoD push
+						 * was skipped unreadable). The push
+						 * page's translation usually lands
+						 * within one service pass (content
+						 * discovery or the host watcher), so
+						 * keep the entry pending for the grace
+						 * window first and only then fall back
+						 * to acknowledging the release.
 						 */
+						if (gsp_sema_fake_rel && ch->engine_type) {
+							if (!ch->defer_since)
+								ch->defer_since =
+									jiffies;
+							if (time_before(jiffies,
+									ch->defer_since +
+									msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
+								svc.gpfifo_nomap++;
+								kfree(pb);
+								break;
+							}
+						}
 if (gsp_sema_fake_rel &&
 					    svc_fake_release(vgpu, ch)) {
+							ch->defer_since = 0;
 							ch->last_get = (idx + 1) %
 								       ch->gpfifo_entries;
 							svc.gpfifo_skipped++;
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 1ffbb4a..234ec2f 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -97,6 +97,12 @@ extern unsigned gsp_sema_ops_log;
  */
 extern unsigned gsp_sema_copy_watch;
 
+/* Data-path watch: 8-byte value matched in copy src/dst and push bytes. */
+extern u64 gsp_sema_copy_pat;
+
+/* CE pending-entry grace window in ms before the release fallback fires. */
+extern unsigned gsp_sema_ce_grace_ms;
+
 /*
  * Grace window (ms) an untranslatable GR GPFIFO entry is left pending before
  * the pump consumes it as stale. The GR push pages are pinned from guest
diff --git a/kernel/main.c b/kernel/main.c
index efa4bf7..7fe2d15 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2466,6 +2466,25 @@ module_param_named(sema_copy_watch, gsp_sema_copy_watch, uint, 0644);
 MODULE_PARM_DESC(sema_copy_watch, "Copy size whose first bytes are logged (default 0)");
 
 /*
+ * Data-path watch: 8-byte value matched against every CE copy's first source
+ * and destination words, and scanned for inside every decoded push, so the
+ * submission carrying a known payload is named even when its size/encoding is
+ * unknown:
+ *   echo 0x0000000400000001 > .../parameters/sema_copy_pat
+ */
+module_param_named(sema_copy_pat, gsp_sema_copy_pat, ullong, 0644);
+MODULE_PARM_DESC(sema_copy_pat, "8-byte pattern matched in copy src/dst and pushes (default 0)");
+
+/*
+ * Grace window (ms) an untranslatable CE entry stays pending before the
+ * fallback acknowledges its release and consumes it. Its work must run before
+ * the release the client synchronizes on; the translation usually lands within
+ * a service pass.
+ */
+module_param_named(sema_ce_grace_ms, gsp_sema_ce_grace_ms, uint, 0644);
+MODULE_PARM_DESC(sema_ce_grace_ms, "CE pending-entry grace window in ms (default 8000)");
+
+/*
  * Grace window (ms) an untranslatable GR entry stays pending before the pump
  * consumes it as stale, so a late push-buffer pin can still make it readable.
  */